enable video capture to support sharing across multiple renderer processes

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include "base/bind.h"
 #include "base/stl_util.h"
 #include "content/browser/browser_thread.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "media/base/yuv_convert.h"
 
-// The number of TransportDIBs VideoCaptureController allocate.
+// The number of DIBs VideoCaptureController allocate.
 static const size_t kNoOfDIBS = 3;
 
+struct VideoCaptureController::ControllerClient {
+  ControllerClient(
+      const VideoCaptureControllerID& id,
+      VideoCaptureControllerEventHandler* handler,
+      base::ProcessHandle render_process,
+      const media::VideoCaptureParams& params)
+      : controller_id(id),
+        event_handler(handler),
+        render_process_handle(render_process),
+        parameters(params),
+        report_ready_to_delete(false) {
+  }
+
+  ~ControllerClient() {}
+
+  // ID used for identifying this object.
+  VideoCaptureControllerID controller_id;
+  VideoCaptureControllerEventHandler* event_handler;
+  // Handle to the render process that will receive the DIBs.
+  base::ProcessHandle render_process_handle;
+  const media::VideoCaptureParams parameters;
+  // Buffers used by this client.
+  std::list<int> buffers;
+  bool report_ready_to_delete;
+};
+
 VideoCaptureController::VideoCaptureController(
-    const VideoCaptureControllerID& id,
-    base::ProcessHandle render_process,
-    VideoCaptureControllerEventHandler* event_handler,
     media_stream::VideoCaptureManager* video_capture_manager)
-    : render_handle_(render_process),
-      report_ready_to_delete_(false),
-      event_handler_(event_handler),
-      id_(id),
-      video_capture_manager_(video_capture_manager) {
-  memset(&params_, 0, sizeof(params_));
+    : frame_info_available_(false),
+      video_capture_manager_(video_capture_manager),
+      device_in_use_(false),
+      state_(media::VideoCapture::kStopped) {
+  memset(&current_params_, 0, sizeof(current_params_));
 }
 
 VideoCaptureController::~VideoCaptureController() {
-  // Delete all TransportDIBs.
+  // Delete all DIBs.
   STLDeleteContainerPairSecondPointers(owned_dibs_.begin(),
                                        owned_dibs_.end());
+  STLDeleteContainerPointers(controller_clients_.begin(),
+                             controller_clients_.end());
 }
 
 void VideoCaptureController::StartCapture(
+    const VideoCaptureControllerID& id,

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

Q: What do you think about moving the resolution and start/stop logic to
VideoCaptureManager and let this class focus on the buffers? I haven't digged
into that, but maybe the logic can be simplified that way. Start/Stop calls can
be relayed to manager and this class allocates memory based on callbacks from
OnFrameInfo? VideoCaptureManager will need to know about all sessionId:s using
the same device and that can be extended to handle resolution and call device
Start/Stop when needed,

[wjia(left Chromium), 2011/10/26 21:17:33]

If we put start/stop logic in manager, we have to move almost everything from
controller to manager, including resolution, device status, buffer status, etc.
That will bloat manager. The current split between manager and controller is
good so that manager handles device open/close/stop/start and controllers, while
controller handles its client (hosts).

+    VideoCaptureControllerEventHandler* event_handler,
+    base::ProcessHandle render_process,
     const media::VideoCaptureParams& params) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-
-  params_ = params;
+  // Signal error in case device is already in error state.
+  if (state_ == media::VideoCapture::kError) {
+    event_handler->OnError(id);
+    return;
+  }
+
+  // Do nothing if this client has called StartCapture before.
+  if (FindClient(id, event_handler, controller_clients_) !=
+      controller_clients_.end() ||
+      FindClient(id, event_handler, pending_clients_) !=
+      pending_clients_.end())
+    return;
+
+  ControllerClient* client = new ControllerClient(id, event_handler,
+                                                  render_process, params);
+  // In case capture has been started, need to check different condtions.
+  if (state_ == media::VideoCapture::kStarted) {
+    // When frame_info has not been returned by device, just wait in pending
+    // queue.
+    if (!frame_info_available_) {
+      pending_clients_.push_back(client);
+      return;
+    }
+
+    // This client has higher resolution than what is currently delivered.
+    // Need restart catpuring.
+    // TODO(wjia): Query the max capability of device. If request is larger
+    // than max, there is no need to restart.
+    if (params.width > frame_info_.width ||
+        params.height > frame_info_.height) {
+      video_capture_manager_->Stop(current_params_.session_id, base::Closure());

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

Q: When is this restarted? I didn't find a call to VideoCaptureManager::Start
after this to restart the device.

[wjia(left Chromium), 2011/10/26 21:17:33]

The restart happens in PostStopping() call. That's when all clients have
returned all buffers and device has been fully stopped.

+      frame_info_available_ = false;
+      state_ = media::VideoCapture::kStopping;
+      pending_clients_.push_back(client);
+      return;
+    }
+
+    // This client's resolution is no larger than what's currently delivered.
+    // Send frame_info and initial buffers to the client.
+    SendFrameInfoAndBuffers(client);
+    controller_clients_.push_back(client);
+    return;
+  }
+
+  // In case the device is in the middle of stopping, put the client in
+  // pending queue.
+  if (state_ == media::VideoCapture::kStopping) {
+    pending_clients_.push_back(client);
+    return;
+  }
+
+  // Fresh start.
+  controller_clients_.push_back(client);
+  current_params_ = params;
   // Order the manager to start the actual capture.
   video_capture_manager_->Start(params, this);
-}
-
-void VideoCaptureController::StopCapture(base::Closure stopped_cb) {
+  state_ = media::VideoCapture::kStarted;
+}
+
+void VideoCaptureController::StopCapture(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    bool force_buffer_return) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  video_capture_manager_->Stop(params_.session_id,
-      base::Bind(&VideoCaptureController::OnDeviceStopped, this, stopped_cb));
-}
-
-void VideoCaptureController::ReturnBuffer(int buffer_id) {
+  ControllerClients::iterator cit;
+  cit = FindClient(id, event_handler, pending_clients_);
+  // If the client is still in pending queue, just remove it.
+  if (cit != pending_clients_.end()) {
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+    pending_clients_.erase(cit);
+    return;
+  }
+
+  cit = FindClient(id, event_handler, controller_clients_);
+  // The client should have called StartCapture.
+  DCHECK(cit != controller_clients_.end());
+
+  if (force_buffer_return) {
+    // The client requests to return buffers which means it can't return
+    // buffers normally. After buffers are returned, client is free to
+    // delete itself. No need to call OnReadyToDelete.
+
+    // Return all buffers held by the clients.
+    for (std::list<int>::iterator bit = (*cit)->buffers.begin();
+         bit != (*cit)->buffers.end(); ++bit) {
+      int buffer_id = *bit;
+      ClientSideDIBCount::iterator dit = client_side_dib_count_.find(buffer_id);
+      if (dit == client_side_dib_count_.end())
+        continue;
+
+      if (--dit->second > 0)
+        continue;
+
+      // Now this |buffer_id| is not used by any client.
+      client_side_dib_count_.erase(buffer_id);
+      {
+        base::AutoLock lock(lock_);
+        free_dibs_.push_back(buffer_id);
+      }
+    }
+    (*cit)->buffers.clear();
+  } else {
+    // Normal way to stop capture.
+    if ((*cit)->buffers.size() > 0) {
+      // There are still some buffers held by the client.
+      (*cit)->report_ready_to_delete = true;
+      return;
+    }
+    // No buffer is held by the client. Ready to delete.
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+  }
+
+  controller_clients_.erase(cit);
+
+  // No more clients. Stop device.
+  if (controller_clients_.size() == 0) {

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

Can there be pending request we should check for, like a new StartCapture has
been called with a larger resolution?

[wjia(left Chromium), 2011/10/26 21:17:33]

StartCapture and StopCapture run on IO thread. There could be pending
StartCapture from another host, but controller doesn't know about it when this
host's StopCapture reaches this point.

+    video_capture_manager_->Stop(current_params_.session_id, base::Closure());

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

Can you explain how the event is used here? Previously was
VideoCaptureManager::Stop called and the controller/host waited until the event
triggered, which happens when the device is stopped and no more frames will be
delivered.

[wjia(left Chromium), 2011/10/26 21:17:33]

Actually, this is a NULL callback. I looked at video_capture_device code.
VideoCaptureDevice::Stop is an asynchronous call (e.g., on linux). When
VideoCaptureManager finishes video_capture_device->Stop(), the device is not
really stopped. So the callback from VideoCaptureManager doesn't tell controller
that it's not in use. The device might call controller during stopping since
controller is the observer in device.

That's also why I added an event function OnDeviceState() in
media::VideoCaptureDevice::EventHandler. That function can tell controller that
the device is really stopped or not.

[perkj_chrome, 2011/10/27 07:27:32]

Please check again. 
All platforms have synchronouse start /stop. On linux the device thread is
started and stopped. 

+    frame_info_available_ = false;
+    state_ = media::VideoCapture::kStopping;
+  }
+}
+
+void VideoCaptureController::ReturnBuffer(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    int buffer_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  bool ready_to_delete;
+  ControllerClients::iterator cit = FindClient(id, event_handler,
+                                               controller_clients_);
+  ClientSideDIBCount::iterator dit = client_side_dib_count_.find(buffer_id);
+  // If this buffer is not held by this client, or this client doesn't exist
+  // in controller, do nothing.
+  if (cit == controller_clients_.end() || dit == client_side_dib_count_.end())
+    return;
+
+  (*cit)->buffers.remove(buffer_id);
+  // If this client has called StopCapture and doesn't hold any buffer after
+  // after this return, ready to delet this client.
+  if ((*cit)->report_ready_to_delete && (*cit)->buffers.size() == 0) {
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+    controller_clients_.erase(cit);
+  }
+  if (--dit->second > 0)
+    return;
+
+  // Now this |buffer_id| is not used by any client.
+  client_side_dib_count_.erase(buffer_id);
   {
     base::AutoLock lock(lock_);
     free_dibs_.push_back(buffer_id);
-    ready_to_delete = (free_dibs_.size() == owned_dibs_.size()) &&
-        report_ready_to_delete_;
-  }
-  if (ready_to_delete) {
-    event_handler_->OnReadyToDelete(id_);
-  }
-}
-
+  }
+
+  // When all buffers have been returned by clients and device has been
+  // called to stop, check if restart is needed. This could happen when
+  // some clients call StopCapture before returning all buffers.
+  if (!ClientHasDIB() && state_ == media::VideoCapture::kStopping) {
+    PostStopping();
+  }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Implements VideoCaptureDevice::EventHandler.
 // OnIncomingCapturedFrame is called the thread running the capture device.
 // I.e.- DirectShow thread on windows and v4l2_thread on Linux.
 void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
                                                      int length,
                                                      base::Time timestamp) {
   int buffer_id = 0;
   base::SharedMemory* dib = NULL;
-  // Check if there is a TransportDIB to fill.
+  // Check if there is a DIB to fill.
   bool buffer_exist = false;
   {
     base::AutoLock lock(lock_);
-    if (!report_ready_to_delete_ && free_dibs_.size() > 0) {
+    if (free_dibs_.size() > 0) {
       buffer_id = free_dibs_.front();
       free_dibs_.pop_front();
       DIBMap::iterator it = owned_dibs_.find(buffer_id);
       if (it != owned_dibs_.end()) {
         dib = it->second;
         buffer_exist = true;
       }
     }
   }
 
@@ skipping 52 matching lines @@
       uint8* vplane = uplane +  (frame_info_.width * frame_info_.height) / 4;
       media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width,
                                frame_info_.height, frame_info_.width * 4,
                                frame_info_.width, frame_info_.width / 2);
       break;
     }
     default:
       NOTREACHED();
   }
 
-  event_handler_->OnBufferReady(id_, buffer_id, timestamp);
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
+                 this, buffer_id, timestamp));
 }
 
 void VideoCaptureController::OnError() {
-  event_handler_->OnError(id_);
-  video_capture_manager_->Error(params_.session_id);
+  video_capture_manager_->Error(current_params_.session_id);
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoErrorOnIOThread, this));
 }
 
 void VideoCaptureController::OnFrameInfo(
     const media::VideoCaptureDevice::Capability& info) {
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoFrameInfoOnIOThread,
+                 this, info));
+}
+
+void VideoCaptureController::OnDeviceState(bool in_use) {
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoDeviceStateOnIOThread, this,
+                 in_use));
+}
+
+void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
+    int buffer_id, base::Time timestamp) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  if (state_ != media::VideoCapture::kStarted) {
+    base::AutoLock lock(lock_);
+    free_dibs_.push_back(buffer_id);
+    return;
+  }
+
+  int count = 0;
+  for (ControllerClients::iterator cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    if ((*cit)->report_ready_to_delete)
+      continue;
+
+    (*cit)->event_handler->OnBufferReady((*cit)->controller_id,
+                                         buffer_id, timestamp);
+    (*cit)->buffers.push_back(buffer_id);
+    count++;
+  }
+  if (count > 0) {
+    client_side_dib_count_[buffer_id] = count;
+  } else {
+    base::AutoLock lock(lock_);
+    free_dibs_.push_back(buffer_id);
+  }
+}
+
+void VideoCaptureController::DoErrorOnIOThread() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  state_ = media::VideoCapture::kError;
+  ControllerClients::iterator cit;
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    (*cit)->event_handler->OnError((*cit)->controller_id);
+  }
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); cit++) {
+    (*cit)->event_handler->OnError((*cit)->controller_id);
+  }
+}
+
+void VideoCaptureController::DoDeviceStateOnIOThread(bool device_in_use) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  device_in_use_ = device_in_use;
+  video_capture_manager_->DeviceStatusFromController(this, device_in_use);
+  if (!device_in_use && state_ == media::VideoCapture::kStopping) {
+    PostStopping();
+  }
+}
+
+void VideoCaptureController::DoFrameInfoOnIOThread(
+    const media::VideoCaptureDevice::Capability info) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK(owned_dibs_.empty());
+
+  // Check if pending clients request higher resolution.
+  current_params_.width = info.width;
+  current_params_.height = info.height;
+  current_params_.frame_per_second = info.frame_rate;
+  ControllerClients::iterator cit;
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); ) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+    controller_clients_.push_back((*cit));
+    pending_clients_.erase(cit++);

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

Doesn't this invalidate the list iterator and shouldn't we use:
cit = pending_clients_.erase(cit);

[wjia(left Chromium), 2011/10/26 21:17:33]

This line is equivalent to
{
ControllerClients::iterator tmp = cit;
++cit;
pending_clients_.erase(tmp);
}
Your suggestion also works.

+  }
+  if (current_params_.width > info.width ||

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

I'd like a comment to explain when this is happening.

[wjia(left Chromium), 2011/10/26 21:17:33]

Done.

+      current_params_.height > info.height) {
+    video_capture_manager_->Stop(current_params_.session_id, base::Closure());
+    frame_info_available_ = false;
+    state_ = media::VideoCapture::kStopping;
+    return;
+  }
+
   bool frames_created = true;
   const size_t needed_size = (info.width * info.height * 3) / 2;
+  base::AutoLock lock(lock_);
   for (size_t i = 1; i <= kNoOfDIBS; ++i) {
     base::SharedMemory* shared_memory = new base::SharedMemory();
     if (!shared_memory->CreateAndMapAnonymous(needed_size)) {
       frames_created = false;
       break;
     }
-    base::SharedMemoryHandle remote_handle;
-    shared_memory->ShareToProcess(render_handle_, &remote_handle);
-
-    base::AutoLock lock(lock_);
     owned_dibs_.insert(std::make_pair(i, shared_memory));
     free_dibs_.push_back(i);
-    event_handler_->OnBufferCreated(id_, remote_handle,
-                                    static_cast<int>(needed_size),
-                                    static_cast<int>(i));
+  }
+  // Check whether all DIBs were created successfully.
+  if (!frames_created) {
+    state_ = media::VideoCapture::kError;
+    for (ControllerClients::iterator it = controller_clients_.begin();
+         it != controller_clients_.end(); it++) {
+      (*it)->event_handler->OnError((*it)->controller_id);
+    }
+    return;
   }
   frame_info_= info;
-
-  // Check that all DIBs where created successfully.
-  if (!frames_created) {
-    event_handler_->OnError(id_);
-  }
-  event_handler_->OnFrameInfo(id_, info.width, info.height, info.frame_rate);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Called by VideoCaptureManager when a device have been stopped.
-// This will report to the event handler that this object is ready to be deleted
-// if all DIBS have been returned.
-void VideoCaptureController::OnDeviceStopped(base::Closure stopped_cb) {
-  bool ready_to_delete_now;
-
-  {
-    base::AutoLock lock(lock_);
-    // Set flag to indicate we need to report when all DIBs have been returned.
-    report_ready_to_delete_ = true;
-    ready_to_delete_now = (free_dibs_.size() == owned_dibs_.size());
-  }
-
-  if (ready_to_delete_now) {
-    event_handler_->OnReadyToDelete(id_);
-  }
-
-  if (!stopped_cb.is_null())
-    stopped_cb.Run();
-}
+  frame_info_available_ = true;
+
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    SendFrameInfoAndBuffersWithLock((*cit), static_cast<int>(needed_size));
+  }
+}
+
+void VideoCaptureController::SendFrameInfoAndBuffers(ControllerClient* client) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(frame_info_available_);
+  const size_t needed_size = (frame_info_.width * frame_info_.height * 3) / 2;
+  base::AutoLock lock(lock_);
+  SendFrameInfoAndBuffersWithLock(client, static_cast<int>(needed_size));
+}
+
+void VideoCaptureController::SendFrameInfoAndBuffersWithLock(
+    ControllerClient* client, int needed_size) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(frame_info_available_);
+  client->event_handler->OnFrameInfo(client->controller_id,
+                                     frame_info_.width, frame_info_.height,
+                                     frame_info_.frame_rate);
+  for (DIBMap::iterator dit = owned_dibs_.begin();
+       dit != owned_dibs_.end(); dit++) {
+    base::SharedMemory* shared_memory = dit->second;
+    int index = dit->first;
+    base::SharedMemoryHandle remote_handle;
+    shared_memory->ShareToProcess(client->render_process_handle,
+                                  &remote_handle);
+    client->event_handler->OnBufferCreated(client->controller_id,
+                                           remote_handle,
+                                           needed_size,
+                                           index);
+  }
+}
+
+void VideoCaptureController::PostStopping() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK_EQ(state_, media::VideoCapture::kStopping);
+
+  if (ClientHasDIB() || device_in_use_)
+    return;
+
+  // It's safe to free all DIB's on IO thread since device won't send
+  // buffer over.
+  free_dibs_.clear();
+  STLDeleteValues(&owned_dibs_);
+
+  if (controller_clients_.size() + pending_clients_.size() == 0) {
+    state_ = media::VideoCapture::kStopped;
+    return;
+  }
+
+  // Restart the device.
+  current_params_.width = 0;
+  current_params_.height = 0;
+  ControllerClients::iterator cit;
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+  }
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); ) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+    controller_clients_.push_back((*cit));
+    pending_clients_.erase(cit++);

[mflodman_chromium_OOO, 2011/10/26 12:34:45]

Same comment about iterator as above.

[wjia(left Chromium), 2011/10/26 21:17:33]

Same explanation as above. :-)

+  }
+  // Request the manager to start the actual capture.
+  video_capture_manager_->Start(current_params_, this);
+  state_ = media::VideoCapture::kStarted;
+}
+
+bool VideoCaptureController::ClientHasDIB() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  for (ClientSideDIBCount::iterator dit = client_side_dib_count_.begin();
+       dit != client_side_dib_count_.end(); dit++) {
+    if (dit->second > 0)
+      return true;
+  }
+  return false;
+}
+
+VideoCaptureController::ControllerClients::iterator
+VideoCaptureController::FindClient(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* handler,
+    ControllerClients& clients) {
+  for (ControllerClients::iterator cit = clients.begin();
+       cit != clients.end(); cit++) {
+    if ((*cit)->controller_id == id && (*cit)->event_handler == handler) {
+      return cit;
+    }
+  }
+  return clients.end();
+}
+