enable video capture to support sharing across multiple renderer processes

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
+#include <stdio.h>

[perkj_chrome, 2011/10/21 14:54:00]

needed?

[wjia(left Chromium), 2011/10/21 23:54:16]

Oops, this is leftover for debugging printf. Removed.

 #include "base/bind.h"
 #include "base/stl_util.h"
 #include "content/browser/browser_thread.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "media/base/yuv_convert.h"
 
-// The number of TransportDIBs VideoCaptureController allocate.
+// The number of DIBs VideoCaptureController allocate.

[perkj_chrome, 2011/10/21 14:54:00]

per host?

[wjia(left Chromium), 2011/10/21 23:54:16]

No. The controller is one per device and allocates only this number of shared
buffers for the browser process. All hosts (i.e., renderer processes) share the
same buffers.

 static const size_t kNoOfDIBS = 3;
 
+struct VideoCaptureController::ControllerClient {
+  ControllerClient(
+      const VideoCaptureControllerID& id,
+      VideoCaptureControllerEventHandler* handler,
+      base::ProcessHandle render_process,
+      const media::VideoCaptureParams& params)
+      : controller_id(id),
+        event_handler(handler),
+        render_process_handle(render_process),
+        parameters(params),
+        report_ready_to_delete(false) {
+  }
+
+  ~ControllerClient() {}
+
+  // ID used for identifying this object.
+  VideoCaptureControllerID controller_id;
+  VideoCaptureControllerEventHandler* event_handler;
+  // Handle to the render process that will receive the DIBs.
+  base::ProcessHandle render_process_handle;
+  const media::VideoCaptureParams parameters;
+  // Buffers used by this client.
+  std::list<int> buffers;
+  bool report_ready_to_delete;
+};
+
 VideoCaptureController::VideoCaptureController(
-    const VideoCaptureControllerID& id,
-    base::ProcessHandle render_process,
-    VideoCaptureControllerEventHandler* event_handler,
     media_stream::VideoCaptureManager* video_capture_manager)
-    : render_handle_(render_process),
-      report_ready_to_delete_(false),
-      event_handler_(event_handler),
-      id_(id),
-      video_capture_manager_(video_capture_manager) {
-  memset(&params_, 0, sizeof(params_));
+    : frame_info_available_(false),
+      video_capture_manager_(video_capture_manager),
+      device_in_use_(false),
+      state_(media::VideoCapture::kStopped) {
+  memset(&current_params_, 0, sizeof(current_params_));
 }
 
 VideoCaptureController::~VideoCaptureController() {
-  // Delete all TransportDIBs.
+  // Delete all DIBs.
   STLDeleteContainerPairSecondPointers(owned_dibs_.begin(),
                                        owned_dibs_.end());
+  STLDeleteContainerPointers(controller_clients_.begin(),
+                             controller_clients_.end());
 }
 
 void VideoCaptureController::StartCapture(

[perkj_chrome, 2011/10/21 14:54:00]

Can you please document the different cases this function handles and how?

[wjia(left Chromium), 2011/10/21 23:54:16]

Comments have been added in each condition.

+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    base::ProcessHandle render_process,
     const media::VideoCaptureParams& params) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  if (state_ == media::VideoCapture::kError) {
+    event_handler->OnError(id);
+    return;
+  }
 
-  params_ = params;
+  if (FindClient(id, event_handler, controller_clients_) !=
+      controller_clients_.end() ||
+      FindClient(id, event_handler, pending_clients_) !=
+      pending_clients_.end())
+    return;

[perkj_chrome, 2011/10/21 14:54:00]

DCHECK? Is this a programming error?

[wjia(left Chromium), 2011/10/21 23:54:16]

I kind of allow this behaviors. Strictly speaking, there should be one
StartCapture per client.

+
+  ControllerClient* client = new ControllerClient(id, event_handler,
+                                                  render_process, params);
+  if (state_ == media::VideoCapture::kStarted) {
+    if (!frame_info_available_) {
+      pending_clients_.push_back(client);
+      return;
+    }
+
+    if (params.width > frame_info_.width ||

[perkj_chrome, 2011/10/21 14:54:00]

You might want to compare with what was actually requested previously (current
params). If the camera can only do vga there is no need to try several times to
open up in 720P.

[wjia(left Chromium), 2011/10/21 23:54:16]

That would be a todo to add code to query the max capability of device. 

+        params.height > frame_info_.height) {
+      video_capture_manager_->Stop(current_params_.session_id, base::Closure());
+      frame_info_available_ = false;
+      state_ = media::VideoCapture::kStopping;
+      pending_clients_.push_back(client);
+      return;
+    }
+
+    SendFrameInfoAndBuffers(client);
+    controller_clients_.push_back(client);
+    return;
+  }
+
+  if (state_ == media::VideoCapture::kStopping) {
+    pending_clients_.push_back(client);
+    return;
+  }
+
+  controller_clients_.push_back(client);
+  current_params_ = params;
   // Order the manager to start the actual capture.
   video_capture_manager_->Start(params, this);
+  state_ = media::VideoCapture::kStarted;
 }
 
-void VideoCaptureController::StopCapture(base::Closure stopped_cb) {
+void VideoCaptureController::StopCapture(

[perkj_chrome, 2011/10/21 14:54:00]

Can you please document what this function is doing and what different cases it
handles?

[wjia(left Chromium), 2011/10/21 23:54:16]

Done.

+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    bool force_buffer_return) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  video_capture_manager_->Stop(params_.session_id,
-      base::Bind(&VideoCaptureController::OnDeviceStopped, this, stopped_cb));
+  ControllerClients::iterator cit;
+  cit = FindClient(id, event_handler, pending_clients_);
+  if (cit != pending_clients_.end()) {
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+    pending_clients_.erase(cit);
+    return;
+  }
+
+  cit = FindClient(id, event_handler, controller_clients_);
+  DCHECK(cit != controller_clients_.end());
+
+  if (force_buffer_return) {
+    for (std::list<int>::iterator bit = (*cit)->buffers.begin();
+         bit != (*cit)->buffers.end(); ++bit) {
+      int buffer_id = *bit;
+      ClientSideDIBCount::iterator dit = client_side_dib_count_.find(buffer_id);
+      if (dit == client_side_dib_count_.end())
+        continue;
+
+      if (--dit->second > 0)
+        continue;
+
+      // Now this |buffer_id| is not used by any client.
+      client_side_dib_count_.erase(buffer_id);
+      {
+        base::AutoLock lock(lock_);
+        free_dibs_.push_back(buffer_id);
+      }
+    }
+    (*cit)->buffers.clear();
+  } else {
+    if ((*cit)->buffers.size() > 0) {
+      (*cit)->report_ready_to_delete = true;
+      return;
+    }
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+  }
+
+  controller_clients_.erase(cit);
+
+  if (controller_clients_.size() == 0) {
+    video_capture_manager_->Stop(current_params_.session_id, base::Closure());
+    frame_info_available_ = false;
+    state_ = media::VideoCapture::kStopping;
+  }
 }
 
-void VideoCaptureController::ReturnBuffer(int buffer_id) {
+void VideoCaptureController::ReturnBuffer(

[perkj_chrome, 2011/10/21 14:54:00]

Please document.

[wjia(left Chromium), 2011/10/21 23:54:16]

Done.

+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* event_handler,
+    int buffer_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
-  bool ready_to_delete;
+  ControllerClients::iterator cit = FindClient(id, event_handler,
+                                               controller_clients_);
+  ClientSideDIBCount::iterator dit = client_side_dib_count_.find(buffer_id);
+  if (cit == controller_clients_.end() || dit == client_side_dib_count_.end())
+    return;
+
+  (*cit)->buffers.remove(buffer_id);
+  if ((*cit)->report_ready_to_delete && (*cit)->buffers.size() == 0) {
+    (*cit)->event_handler->OnReadyToDelete((*cit)->controller_id);
+    controller_clients_.erase(cit);
+  }
+  if (--dit->second > 0)
+    return;
+
+  // Now this |buffer_id| is not used by any client.
+  client_side_dib_count_.erase(buffer_id);
   {
     base::AutoLock lock(lock_);
     free_dibs_.push_back(buffer_id);
-    ready_to_delete = (free_dibs_.size() == owned_dibs_.size()) &&
-        report_ready_to_delete_;
   }
-  if (ready_to_delete) {
-    event_handler_->OnReadyToDelete(id_);
+
+  if (!ClientHasDIB() && state_ == media::VideoCapture::kStopping) {
+    PostStopping();
   }
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 // Implements VideoCaptureDevice::EventHandler.
 // OnIncomingCapturedFrame is called the thread running the capture device.
 // I.e.- DirectShow thread on windows and v4l2_thread on Linux.
 void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
                                                      int length,
                                                      base::Time timestamp) {
   int buffer_id = 0;
   base::SharedMemory* dib = NULL;
-  // Check if there is a TransportDIB to fill.
+  // Check if there is a DIB to fill.
   bool buffer_exist = false;
   {
     base::AutoLock lock(lock_);
-    if (!report_ready_to_delete_ && free_dibs_.size() > 0) {
+    if (free_dibs_.size() > 0) {
       buffer_id = free_dibs_.front();
       free_dibs_.pop_front();
       DIBMap::iterator it = owned_dibs_.find(buffer_id);
       if (it != owned_dibs_.end()) {
         dib = it->second;
         buffer_exist = true;
       }
     }
   }
 
@@ skipping 52 matching lines @@
       uint8* vplane = uplane +  (frame_info_.width * frame_info_.height) / 4;
       media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width,
                                frame_info_.height, frame_info_.width * 4,
                                frame_info_.width, frame_info_.width / 2);
       break;
     }
     default:
       NOTREACHED();
   }
 
-  event_handler_->OnBufferReady(id_, buffer_id, timestamp);
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
+                 this, buffer_id, timestamp));
 }
 
 void VideoCaptureController::OnError() {
-  event_handler_->OnError(id_);
-  video_capture_manager_->Error(params_.session_id);
+  video_capture_manager_->Error(current_params_.session_id);
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoErrorOnIOThread, this));
 }
 
 void VideoCaptureController::OnFrameInfo(
     const media::VideoCaptureDevice::Capability& info) {
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoFrameInfoOnIOThread,
+                 this, info));
+}
+
+void VideoCaptureController::OnDeviceState(bool in_use) {
+  BrowserThread::PostTask(BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoDeviceStateOnIOThread, this,
+                 in_use));
+}
+
+void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
+    int buffer_id, base::Time timestamp) {

[perkj_chrome, 2011/10/21 14:54:00]

You say that this is of type int here but in OnBufferReady you say it is of type
 base::SharedMemoryHandle.
Please use the same type name everywhere. I prefer  base::SharedMemoryHandle if
that is actually what it is. 

[wjia(left Chromium), 2011/10/21 23:54:16]

I guess you are confused with OnBufferReady and OnBufferCreated. OnBufferReady
is using "int", while OnBufferCreated is using both "int" and
"base::SharedMemoryHandle" since it needs to pass the pair to renderer process.
We are using buffer_id after that.

+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  if (state_ != media::VideoCapture::kStarted) {
+    base::AutoLock lock(lock_);
+    free_dibs_.push_back(buffer_id);
+    return;
+  }
+
+  int count = 0;
+  for (ControllerClients::iterator cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    if ((*cit)->report_ready_to_delete)
+      continue;
+
+    (*cit)->event_handler->OnBufferReady((*cit)->controller_id,
+                                         buffer_id, timestamp);
+    (*cit)->buffers.push_back(buffer_id);
+    count++;
+  }
+  if (count > 0) {
+    client_side_dib_count_[buffer_id] = count;
+  } else {
+    base::AutoLock lock(lock_);
+    free_dibs_.push_back(buffer_id);
+  }
+}
+
+void VideoCaptureController::DoErrorOnIOThread() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  state_ = media::VideoCapture::kError;
+  ControllerClients::iterator cit;
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    (*cit)->event_handler->OnError((*cit)->controller_id);
+  }
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); cit++) {
+    (*cit)->event_handler->OnError((*cit)->controller_id);
+  }
+}
+
+void VideoCaptureController::DoDeviceStateOnIOThread(bool device_in_use) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  device_in_use_ = device_in_use;
+  video_capture_manager_->DeviceStatusFromController(this, device_in_use);
+  if (!device_in_use && state_ == media::VideoCapture::kStopping) {
+    PostStopping();
+  }
+}
+
+void VideoCaptureController::DoFrameInfoOnIOThread(
+    const media::VideoCaptureDevice::Capability info) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK(owned_dibs_.empty());
+
+  // Check if pending clients request higher resolution.
+  current_params_.width = info.width;
+  current_params_.height = info.height;
+  current_params_.frame_per_second = info.frame_rate;
+  ControllerClients::iterator cit;
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); ) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+    controller_clients_.push_back((*cit));
+    pending_clients_.erase(cit++);
+  }
+  if (current_params_.width > info.width ||
+      current_params_.height > info.height) {
+    video_capture_manager_->Stop(current_params_.session_id, base::Closure());
+    frame_info_available_ = false;
+    state_ = media::VideoCapture::kStopping;
+    return;
+  }
+
   bool frames_created = true;
   const size_t needed_size = (info.width * info.height * 3) / 2;
+  base::AutoLock lock(lock_);
   for (size_t i = 1; i <= kNoOfDIBS; ++i) {
     base::SharedMemory* shared_memory = new base::SharedMemory();
     if (!shared_memory->CreateAndMapAnonymous(needed_size)) {
       frames_created = false;
       break;
     }
-    base::SharedMemoryHandle remote_handle;
-    shared_memory->ShareToProcess(render_handle_, &remote_handle);
-
-    base::AutoLock lock(lock_);
     owned_dibs_.insert(std::make_pair(i, shared_memory));
     free_dibs_.push_back(i);
-    event_handler_->OnBufferCreated(id_, remote_handle,
-                                    static_cast<int>(needed_size),
-                                    static_cast<int>(i));
+  }
+  // Check whether all DIBs were created successfully.
+  if (!frames_created) {
+    state_ = media::VideoCapture::kError;
+    for (ControllerClients::iterator it = controller_clients_.begin();
+         it != controller_clients_.end(); it++) {
+      (*it)->event_handler->OnError((*it)->controller_id);
+    }
+    return;
   }
   frame_info_= info;
-
-  // Check that all DIBs where created successfully.
-  if (!frames_created) {
-    event_handler_->OnError(id_);
-  }
-  event_handler_->OnFrameInfo(id_, info.width, info.height, info.frame_rate);
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// Called by VideoCaptureManager when a device have been stopped.
-// This will report to the event handler that this object is ready to be deleted
-// if all DIBS have been returned.
-void VideoCaptureController::OnDeviceStopped(base::Closure stopped_cb) {
-  bool ready_to_delete_now;
-
-  {
-    base::AutoLock lock(lock_);
-    // Set flag to indicate we need to report when all DIBs have been returned.
-    report_ready_to_delete_ = true;
-    ready_to_delete_now = (free_dibs_.size() == owned_dibs_.size());
-  }
-
-  if (ready_to_delete_now) {
-    event_handler_->OnReadyToDelete(id_);
-  }
-
-  if (!stopped_cb.is_null())
-    stopped_cb.Run();
-}
+  frame_info_available_ = true;
+
+  for (cit = controller_clients_.begin();

[perkj_chrome, 2011/10/21 14:54:00]

Use SendFrameInfoAndBuffers - avoid duplicate code.

[wjia(left Chromium), 2011/10/21 23:54:16]

Done.

+       cit != controller_clients_.end(); cit++) {
+    (*cit)->event_handler->OnFrameInfo((*cit)->controller_id,
+                                       info.width, info.height,
+                                       info.frame_rate);
+    for (DIBMap::iterator dit = owned_dibs_.begin();
+         dit != owned_dibs_.end(); dit++) {
+      int index = dit->first;
+      base::SharedMemory* shared_memory = dit->second;
+      base::SharedMemoryHandle remote_handle;
+      shared_memory->ShareToProcess((*cit)->render_process_handle,
+                                    &remote_handle);
+      (*cit)->event_handler->OnBufferCreated((*cit)->controller_id,
+                                             remote_handle,
+                                             static_cast<int>(needed_size),
+                                             index);
+    }
+  }
+}
+
+void VideoCaptureController::SendFrameInfoAndBuffers(ControllerClient* client) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(frame_info_available_);
+  const size_t needed_size = (frame_info_.width * frame_info_.height * 3) / 2;

[perkj_chrome, 2011/10/21 14:54:00]

not used

[wjia(left Chromium), 2011/10/21 23:54:16]

It's used in OnBufferCreated.

+  client->event_handler->OnFrameInfo(client->controller_id,
+                                     frame_info_.width, frame_info_.height,
+                                     frame_info_.frame_rate);
+  base::AutoLock lock(lock_);
+  for (DIBMap::iterator dit = owned_dibs_.begin();
+       dit != owned_dibs_.end(); dit++) {
+    base::SharedMemory* shared_memory = dit->second;
+    int index = dit->first;
+    base::SharedMemoryHandle remote_handle;
+    shared_memory->ShareToProcess(client->render_process_handle,
+                                  &remote_handle);
+    client->event_handler->OnBufferCreated(client->controller_id,
+                                           remote_handle,
+                                           static_cast<int>(needed_size),
+                                           index);
+  }
+}
+
+void VideoCaptureController::PostStopping() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK_EQ(state_, media::VideoCapture::kStopping);
+
+  if (ClientHasDIB() || device_in_use_)
+    return;
+
+  // It's safe to free all DIB's on IO thread since device won't send
+  // buffer over.
+  free_dibs_.clear();
+  STLDeleteValues(&owned_dibs_);
+
+  if (controller_clients_.size() + pending_clients_.size() == 0) {
+    state_ = media::VideoCapture::kStopped;
+    return;
+  }
+
+  // Restart the device.
+  current_params_.width = 0;
+  current_params_.height = 0;
+  ControllerClients::iterator cit;
+  for (cit = controller_clients_.begin();
+       cit != controller_clients_.end(); cit++) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+  }
+  for (cit = pending_clients_.begin();
+       cit != pending_clients_.end(); ) {
+    if (current_params_.width < (*cit)->parameters.width)
+      current_params_.width = (*cit)->parameters.width;
+    if (current_params_.height < (*cit)->parameters.height)
+      current_params_.height = (*cit)->parameters.height;
+    controller_clients_.push_back((*cit));
+    pending_clients_.erase(cit++);
+  }
+  // Request the manager to start the actual capture.
+  video_capture_manager_->Start(current_params_, this);
+  state_ = media::VideoCapture::kStarted;
+}
+
+bool VideoCaptureController::ClientHasDIB() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  for (ClientSideDIBCount::iterator dit = client_side_dib_count_.begin();
+       dit != client_side_dib_count_.end(); dit++) {
+    if (dit->second > 0)
+      return true;
+  }
+  return false;
+}
+
+VideoCaptureController::ControllerClients::iterator
+VideoCaptureController::FindClient(
+    const VideoCaptureControllerID& id,
+    VideoCaptureControllerEventHandler* handler,
+    ControllerClients& clients) {
+  for (ControllerClients::iterator cit = clients.begin();
+       cit != clients.end(); cit++) {
+    if ((*cit)->controller_id == id && (*cit)->event_handler == handler) {
+      return cit;
+    }
+  }
+  return clients.end();
+}
+