enable video capture to support sharing across multiple renderer processes

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include "base/bind.h"
 #include "base/stl_util.h"
 #include "content/browser/browser_thread.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "media/base/yuv_convert.h"
 
-// The number of TransportDIBs VideoCaptureController allocate.
+// The number of DIBs VideoCaptureController allocate.
 static const size_t kNoOfDIBS = 3;
 
+struct VideoCaptureController::ControllerClient {
+  ControllerClient(
+      const VideoCaptureControllerID& id,
+      VideoCaptureControllerEventHandler* handler,
+      base::ProcessHandle render_process,
+      const media::VideoCaptureParams& params)
+      : controller_id(id),
+        event_handler(handler),
+        render_process_handle(render_process),
+        parameters(params),
+        report_ready_to_delete(false) {
+  }
+
+  ~ControllerClient() {}
+
+  // ID used for identifying this object.
+  VideoCaptureControllerID controller_id;
+  VideoCaptureControllerEventHandler* event_handler;
+  // Handle to the render process that will receive the DIBs.
+  base::ProcessHandle render_process_handle;
+  const media::VideoCaptureParams parameters;
+  // Buffers used by this client.
+  std::list<int> buffers;
+  bool report_ready_to_delete;
+};
+
 VideoCaptureController::VideoCaptureController(
-    const VideoCaptureControllerID& id,
-    base::ProcessHandle render_process,
-    VideoCaptureControllerEventHandler* event_handler,
     media_stream::VideoCaptureManager* video_capture_manager)
-    : render_handle_(render_process),
-      report_ready_to_delete_(false),
-      event_handler_(event_handler),
-      id_(id),
-      video_capture_manager_(video_capture_manager) {
-  memset(&params_, 0, sizeof(params_));
+    : frame_info_available_(false),
+      video_capture_manager_(video_capture_manager),
+      device_in_use_(false),
+      state_(media::VideoCapture::kStopped) {
+  memset(&current_params_, 0, sizeof(current_params_));
 }
 
 VideoCaptureController::~VideoCaptureController() {
-  // Delete all TransportDIBs.
+  // Delete all DIBs.
   STLDeleteContainerPairSecondPointers(owned_dibs_.begin(),
                                        owned_dibs_.end());
+  STLDeleteContainerPointers(controller_clients_.begin(),
+                             controller_clients_.end());
 }
 
 void VideoCaptureController::StartCapture(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    base::ProcessHandle render_process,
     const media::VideoCaptureParams& params) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-
-  params_ = params;
+  // Signal error in case device is already in error state.
+  if (state_ == media::VideoCapture::kError) {
+    event_handler->OnError(id);
+    return;
+  }
+
+  // Do nothing if this client has called StartCapture before.
+  if (FindClient(id, event_handler, controller_clients_) !=
+      controller_clients_.end() ||
+      FindClient(id, event_handler, pending_clients_) !=
+      pending_clients_.end())
+    return;
+
+  ControllerClient* client = new ControllerClient(id, event_handler,
+                                                  render_process, params);
+  // In case capture has been started, need to check different condtions.
+  if (state_ == media::VideoCapture::kStarted) {
+    // When frame_info has not been returned by device, just wait in pending
+    // queue.
+    if (!frame_info_available_) {
+      pending_clients_.push_back(client);
+      return;
+    }
+
+    // This client has higher resolution than what is currently delivered.
+    // Need restart catpuring.

[perkj_chrome, 2011/10/27 14:30:30]

nit: catpuring- capturing.

[wjia(left Chromium), 2011/10/28 01:27:40]

Done.

+    // TODO(wjia): Query the max capability of device. If request is larger

[perkj_chrome, 2011/10/27 14:30:30]

Don't query the device. (it does not work on Mac and work poor on Linux) I meant
that you should remember what max have been requested before. You never decrease
the resolution so you can use that here instead of frame_info_ which seems to be
what the camera actually produces at the moment?

[wjia(left Chromium), 2011/10/28 01:27:40]

Done. For now, we don't decrease resolution. In the future, we might want to do
that for the sake of saving on computing and memeory.

+    // than max, there is no need to restart.
+    if (params.width > frame_info_.width ||
+        params.height > frame_info_.height) {
+      video_capture_manager_->Stop(current_params_.session_id, base::Closure());

[perkj_chrome, 2011/10/27 14:30:30]

Use base::Closure instead of your callback.

[wjia(left Chromium), 2011/10/28 01:27:40]

Done. I assume you mean "Use your callback, instead of base::Closure".

+      frame_info_available_ = false;
+      state_ = media::VideoCapture::kStopping;
+      pending_clients_.push_back(client);
+      return;
+    }
+
+    // This client's resolution is no larger than what's currently delivered.
+    // Send frame_info and initial buffers to the client.
+    SendFrameInfoAndBuffers(client);
+    controller_clients_.push_back(client);
+    return;
+  }
+
+  // In case the device is in the middle of stopping, put the client in
+  // pending queue.
+  if (state_ == media::VideoCapture::kStopping) {
+    pending_clients_.push_back(client);
+    return;
+  }
+
+  // Fresh start.
+  controller_clients_.push_back(client);
+  current_params_ = params;
   // Order the manager to start the actual capture.
   video_capture_manager_->Start(params, this);
-}
-
-void VideoCaptureController::StopCapture(base::Closure stopped_cb) {
+  state_ = media::VideoCapture::kStarted;
+}
+
+void VideoCaptureController::StopCapture(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    bool force_buffer_return) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  video_capture_manager_->Stop(params_.session_id,
-      base::Bind(&VideoCaptureController::OnDeviceStopped, this, stopped_cb));
-}
-
-void VideoCaptureController::ReturnBuffer(int buffer_id) {
+  ControllerClients::iterator cit;
+  cit = FindClient(id, event_handler, pending_clients_);
+  // If the client is still in pending queue, just remove it.
+  if (cit != pending_clients_.end()) {
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+    pending_clients_.erase(cit);
+    return;
+  }
+
+  cit = FindClient(id, event_handler, controller_clients_);
+  // The client should have called StartCapture.
+  DCHECK(cit != controller_clients_.end());
+
+  if (force_buffer_return) {
+    // The client requests to return buffers which means it can't return
+    // buffers normally. After buffers are returned, client is free to
+    // delete itself. No need to call OnReadyToDelete.
+
+    // Return all buffers held by the clients.
+    for (std::list<int>::iterator bit = (*cit)->buffers.begin();
+         bit != (*cit)->buffers.end(); ++bit) {
+      int buffer_id = *bit;
+      ClientSideDIBCount::iterator dit = client_side_dib_count_.find(buffer_id);
+      if (dit == client_side_dib_count_.end())
+        continue;
+
+      if (--dit->second > 0)
+        continue;
+
+      // Now this |buffer_id| is not used by any client.
+      client_side_dib_count_.erase(buffer_id);
+      {
+        base::AutoLock lock(lock_);
+        free_dibs_.push_back(buffer_id);
+      }
+    }
+    (*cit)->buffers.clear();
+  } else {
+    // Normal way to stop capture.
+    if ((*cit)->buffers.size() > 0) {
+      // There are still some buffers held by the client.
+      (*cit)->report_ready_to_delete = true;
+      return;
+    }
+    // No buffer is held by the client. Ready to delete.
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+  }
+
+  controller_clients_.erase(cit);
+
+  // No more clients. Stop device.
+  if (controller_clients_.size() == 0) {
+    video_capture_manager_->Stop(current_params_.session_id, base::Closure());
+    frame_info_available_ = false;
+    state_ = media::VideoCapture::kStopping;
+  }
+}
+
+void VideoCaptureController::ReturnBuffer(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    int buffer_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  bool ready_to_delete;
+  ControllerClients::iterator cit = FindClient(id, event_handler,
+                                               controller_clients_);
+  ClientSideDIBCount::iterator dit = client_side_dib_count_.find(buffer_id);
+  // If this buffer is not held by this client, or this client doesn't exist
+  // in controller, do nothing.
+  if (cit == controller_clients_.end() || dit == client_side_dib_count_.end())
+    return;
+
+  (*cit)->buffers.remove(buffer_id);
+  // If this client has called StopCapture and doesn't hold any buffer after
+  // after this return, ready to delet this client.
+  if ((*cit)->report_ready_to_delete && (*cit)->buffers.size() == 0) {
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+    controller_clients_.erase(cit);
+  }
+  if (--dit->second > 0)
+    return;
+
+  // Now this |buffer_id| is not used by any client.
+  client_side_dib_count_.erase(buffer_id);
   {
     base::AutoLock lock(lock_);
     free_dibs_.push_back(buffer_id);
-    ready_to_delete = (free_dibs_.size() == owned_dibs_.size()) &&
-        report_ready_to_delete_;
-  }
-  if (ready_to_delete) {
-    event_handler_->OnReadyToDelete(id_);
-  }
-}
-
+  }
+
+  // When all buffers have been returned by clients and device has been
+  // called to stop, check if restart is needed. This could happen when
+  // some clients call StopCapture before returning all buffers.
+  if (!ClientHasDIB() && state_ == media::VideoCapture::kStopping) {
+    PostStopping();
+  }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Implements VideoCaptureDevice::EventHandler.
 // OnIncomingCapturedFrame is called the thread running the capture device.
 // I.e.- DirectShow thread on windows and v4l2_thread on Linux.
 void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
                                                      int length,
                                                      base::Time timestamp) {
   int buffer_id = 0;
   base::SharedMemory* dib = NULL;
-  // Check if there is a TransportDIB to fill.
+  // Check if there is a DIB to fill.
   bool buffer_exist = false;
   {
     base::AutoLock lock(lock_);
-    if (!report_ready_to_delete_ && free_dibs_.size() > 0) {
+    if (free_dibs_.size() > 0) {
       buffer_id = free_dibs_.front();
       free_dibs_.pop_front();
       DIBMap::iterator it = owned_dibs_.find(buffer_id);
       if (it != owned_dibs_.end()) {
         dib = it->second;
         buffer_exist = true;
       }
     }
   }
 
@@ skipping 52 matching lines @@
       uint8* vplane = uplane +  (frame_info_.width * frame_info_.height) / 4;
       media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width,
                                frame_info_.height, frame_info_.width * 4,
                                frame_info_.width, frame_info_.width / 2);
       break;
     }
     default:
       NOTREACHED();
   }
 
-  event_handler_->OnBufferReady(id_, buffer_id, timestamp);
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
+                 this, buffer_id, timestamp));
 }
 
 void VideoCaptureController::OnError() {
-  event_handler_->OnError(id_);
-  video_capture_manager_->Error(params_.session_id);
+  video_capture_manager_->Error(current_params_.session_id);
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoErrorOnIOThread, this));
 }
 
 void VideoCaptureController::OnFrameInfo(
     const media::VideoCaptureDevice::Capability& info) {
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoFrameInfoOnIOThread,
+                 this, info));
+}
+
+void VideoCaptureController::OnDeviceState(bool in_use) {
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoDeviceStateOnIOThread, this,
+                 in_use));
+}
+
+void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
+    int buffer_id, base::Time timestamp) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  if (state_ != media::VideoCapture::kStarted) {
+    base::AutoLock lock(lock_);
+    free_dibs_.push_back(buffer_id);
+    return;
+  }
+
+  int count = 0;
+  for (ControllerClients::iterator cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    if ((*cit)->report_ready_to_delete)
+      continue;
+
+    (*cit)->event_handler->OnBufferReady((*cit)->controller_id,
+                                         buffer_id, timestamp);
+    (*cit)->buffers.push_back(buffer_id);
+    count++;
+  }
+  if (count > 0) {
+    client_side_dib_count_[buffer_id] = count;
+  } else {
+    base::AutoLock lock(lock_);
+    free_dibs_.push_back(buffer_id);
+  }
+}
+
+void VideoCaptureController::DoErrorOnIOThread() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  state_ = media::VideoCapture::kError;
+  ControllerClients::iterator cit;
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    (*cit)->event_handler->OnError((*cit)->controller_id);
+  }
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); cit++) {
+    (*cit)->event_handler->OnError((*cit)->controller_id);
+  }
+}
+
+void VideoCaptureController::DoDeviceStateOnIOThread(bool device_in_use) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  device_in_use_ = device_in_use;
+  video_capture_manager_->DeviceStatusFromController(this, device_in_use);
+  if (!device_in_use && state_ == media::VideoCapture::kStopping) {
+    PostStopping();
+  }
+}
+
+void VideoCaptureController::DoFrameInfoOnIOThread(
+    const media::VideoCaptureDevice::Capability info) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK(owned_dibs_.empty());
+
+  // Check if pending clients request higher resolution.
+  current_params_.width = info.width;
+  current_params_.height = info.height;
+  current_params_.frame_per_second = info.frame_rate;
+  ControllerClients::iterator cit;
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); ) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+    controller_clients_.push_back((*cit));
+    pending_clients_.erase(cit++);
+  }
+  // Take care of the case where a new client calls StartCapture with higher
+  // resolution before DoFrameInfoOnIOThread (of previous client) is called.
+  if (current_params_.width > info.width ||

[perkj_chrome, 2011/10/27 14:30:30]

You should not trust that the camera give you what you requested.
You are not guaranteed to get he frame size you request. It might be smaller, it
might bee larger. It is up to the camera drivers.

[wjia(left Chromium), 2011/10/28 01:27:40]

Done.

+      current_params_.height > info.height) {
+    video_capture_manager_->Stop(current_params_.session_id, base::Closure());
+    frame_info_available_ = false;
+    state_ = media::VideoCapture::kStopping;
+    return;
+  }
+
   bool frames_created = true;
   const size_t needed_size = (info.width * info.height * 3) / 2;
+  base::AutoLock lock(lock_);
   for (size_t i = 1; i <= kNoOfDIBS; ++i) {
     base::SharedMemory* shared_memory = new base::SharedMemory();
     if (!shared_memory->CreateAndMapAnonymous(needed_size)) {
       frames_created = false;
       break;
     }
-    base::SharedMemoryHandle remote_handle;
-    shared_memory->ShareToProcess(render_handle_, &remote_handle);
-
-    base::AutoLock lock(lock_);
     owned_dibs_.insert(std::make_pair(i, shared_memory));
     free_dibs_.push_back(i);
-    event_handler_->OnBufferCreated(id_, remote_handle,
-                                    static_cast<int>(needed_size),
-                                    static_cast<int>(i));
+  }
+  // Check whether all DIBs were created successfully.
+  if (!frames_created) {
+    state_ = media::VideoCapture::kError;
+    for (ControllerClients::iterator it = controller_clients_.begin();
+         it != controller_clients_.end(); it++) {
+      (*it)->event_handler->OnError((*it)->controller_id);
+    }
+    return;
   }
   frame_info_= info;
-
-  // Check that all DIBs where created successfully.
-  if (!frames_created) {
-    event_handler_->OnError(id_);
-  }
-  event_handler_->OnFrameInfo(id_, info.width, info.height, info.frame_rate);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Called by VideoCaptureManager when a device have been stopped.
-// This will report to the event handler that this object is ready to be deleted
-// if all DIBS have been returned.
-void VideoCaptureController::OnDeviceStopped(base::Closure stopped_cb) {
-  bool ready_to_delete_now;
-
-  {
-    base::AutoLock lock(lock_);
-    // Set flag to indicate we need to report when all DIBs have been returned.
-    report_ready_to_delete_ = true;
-    ready_to_delete_now = (free_dibs_.size() == owned_dibs_.size());
-  }
-
-  if (ready_to_delete_now) {
-    event_handler_->OnReadyToDelete(id_);
-  }
-
-  if (!stopped_cb.is_null())
-    stopped_cb.Run();
-}
+  frame_info_available_ = true;
+
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    SendFrameInfoAndBuffersWithLock((*cit), static_cast<int>(needed_size));
+  }
+}
+
+void VideoCaptureController::SendFrameInfoAndBuffers(ControllerClient* client) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(frame_info_available_);
+  const size_t needed_size = (frame_info_.width * frame_info_.height * 3) / 2;
+  base::AutoLock lock(lock_);
+  SendFrameInfoAndBuffersWithLock(client, static_cast<int>(needed_size));
+}
+
+void VideoCaptureController::SendFrameInfoAndBuffersWithLock(
+    ControllerClient* client, int needed_size) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(frame_info_available_);
+  client->event_handler->OnFrameInfo(client->controller_id,
+                                     frame_info_.width, frame_info_.height,
+                                     frame_info_.frame_rate);
+  for (DIBMap::iterator dit = owned_dibs_.begin();
+       dit != owned_dibs_.end(); dit++) {
+    base::SharedMemory* shared_memory = dit->second;
+    int index = dit->first;
+    base::SharedMemoryHandle remote_handle;
+    shared_memory->ShareToProcess(client->render_process_handle,
+                                  &remote_handle);
+    client->event_handler->OnBufferCreated(client->controller_id,
+                                           remote_handle,
+                                           needed_size,
+                                           index);
+  }
+}
+
+void VideoCaptureController::PostStopping() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK_EQ(state_, media::VideoCapture::kStopping);
+
+  if (ClientHasDIB() || device_in_use_)
+    return;
+
+  // It's safe to free all DIB's on IO thread since device won't send
+  // buffer over.
+  free_dibs_.clear();
+  STLDeleteValues(&owned_dibs_);
+
+  if (controller_clients_.size() + pending_clients_.size() == 0) {
+    state_ = media::VideoCapture::kStopped;
+    return;
+  }
+
+  // Restart the device.
+  current_params_.width = 0;
+  current_params_.height = 0;
+  ControllerClients::iterator cit;
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+  }
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); ) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+    controller_clients_.push_back((*cit));
+    pending_clients_.erase(cit++);
+  }
+  // Request the manager to start the actual capture.
+  video_capture_manager_->Start(current_params_, this);
+  state_ = media::VideoCapture::kStarted;
+}
+
+bool VideoCaptureController::ClientHasDIB() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  for (ClientSideDIBCount::iterator dit = client_side_dib_count_.begin();
+       dit != client_side_dib_count_.end(); dit++) {
+    if (dit->second > 0)
+      return true;
+  }
+  return false;
+}
+
+VideoCaptureController::ControllerClients::iterator
+VideoCaptureController::FindClient(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* handler,
+    ControllerClients& clients) {
+  for (ControllerClients::iterator cit = clients.begin();
+       cit != clients.end(); cit++) {
+    if ((*cit)->controller_id == id && (*cit)->event_handler == handler) {
+      return cit;
+    }
+  }
+  return clients.end();
+}
+