[Mojo Video Capture] Split OnIncomingCapturedVideoFrame() to OnNewBuffer() + OnFrameReadyInBuffer()

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
 #include <set>
 
 #include "base/bind.h"
 #include "base/command_line.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "build/build_config.h"
 #include "components/display_compositor/gl_helper.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
+#include "content/common/video_capture.mojom.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/common/content_switches.h"
 #include "media/base/video_frame.h"
 #include "media/capture/video/video_capture_buffer_pool.h"
 #include "media/capture/video/video_capture_buffer_tracker_factory_impl.h"
 #include "media/capture/video/video_capture_device_client.h"
 #include "mojo/public/cpp/system/platform_handle.h"
 
 #if !defined(OS_ANDROID)
 #include "content/browser/compositor/image_transport_factory.h"
 #endif
 
 using media::VideoCaptureFormat;
 using media::VideoFrame;
 using media::VideoFrameMetadata;
 
 namespace content {
 
 namespace {
 
 static const int kInfiniteRatio = 99999;
 
 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
     UMA_HISTOGRAM_SPARSE_SLOWLY( \
         name, \
         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
 
+// This manual copy routine is needed because base::DictionaryValue does not
+// allow the default copy operator.
+media::mojom::VideoFrameInfoPtr CloneFrameInfo(
+    media::mojom::VideoFrameInfoPtr* frame_info) {
+  auto result = media::mojom::VideoFrameInfo::New();
+  result->timestamp = (*frame_info)->timestamp;
+  result->pixel_format = (*frame_info)->pixel_format;
+  result->storage_type = (*frame_info)->storage_type;
+  result->coded_size = (*frame_info)->coded_size;
+  result->visible_rect = (*frame_info)->visible_rect;
+  result->metadata = base::MakeUnique<base::DictionaryValue>();
+  result->metadata->MergeDictionary((*frame_info)->metadata.get());
+  return result;
+}
+
 }  // anonymous namespace
 
 struct VideoCaptureController::ControllerClient {
   ControllerClient(VideoCaptureControllerID id,
                    VideoCaptureControllerEventHandler* handler,
                    media::VideoCaptureSessionId session_id,
                    const media::VideoCaptureParams& params)
       : controller_id(id),
         event_handler(handler),
         session_id(session_id),
@@ skipping 28 matching lines @@
   // simplify the code in both places.
   bool session_closed;
 
   // Indicates whether the client is paused, if true, VideoCaptureController
   // stops updating its buffer.
   bool paused;
 };
 
 VideoCaptureController::BufferState::BufferState(
     int buffer_id,
-    int frame_feedback_id,
     media::VideoFrameConsumerFeedbackObserver* consumer_feedback_observer,
-    media::FrameBufferPool* frame_buffer_pool)
+    mojo::ScopedSharedBufferHandle handle)
     : buffer_id_(buffer_id),
-      frame_feedback_id_(frame_feedback_id),
+      frame_feedback_id_(0),
       consumer_feedback_observer_(consumer_feedback_observer),
-      frame_buffer_pool_(frame_buffer_pool),
+      buffer_handle_(std::move(handle)),
       max_consumer_utilization_(
           media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded),
       consumer_hold_count_(0) {}
 
 VideoCaptureController::BufferState::~BufferState() = default;
 
 VideoCaptureController::BufferState::BufferState(
-    const VideoCaptureController::BufferState& other) = default;
+    VideoCaptureController::BufferState&& other) = default;
+
+VideoCaptureController::BufferState& VideoCaptureController::BufferState::
+operator=(BufferState&& other) = default;
+
+void VideoCaptureController::BufferState::set_read_permission(
+    std::unique_ptr<media::Ownership> buffer_read_permission) {
+  buffer_read_permission_ = std::move(buffer_read_permission);
+}
+
+void VideoCaptureController::BufferState::set_frame_feedback_id(
+    int frame_feedback_id) {
+  frame_feedback_id_ = frame_feedback_id;
+}
 
 void VideoCaptureController::BufferState::RecordConsumerUtilization(
     double utilization) {
   if (std::isfinite(utilization) && utilization >= 0.0) {
     max_consumer_utilization_ =
         std::max(max_consumer_utilization_, utilization);
   }
 }
 
 void VideoCaptureController::BufferState::IncreaseConsumerCount() {
-  if (consumer_hold_count_ == 0)
-    if (frame_buffer_pool_ != nullptr)
-      frame_buffer_pool_->SetBufferHold(buffer_id_);
   consumer_hold_count_++;
 }
 
 void VideoCaptureController::BufferState::DecreaseConsumerCount() {
+  LOG(ERROR) << "1";
   consumer_hold_count_--;
   if (consumer_hold_count_ == 0) {
     if (consumer_feedback_observer_ != nullptr &&
         max_consumer_utilization_ !=
             media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded) {
       consumer_feedback_observer_->OnUtilizationReport(
           frame_feedback_id_, max_consumer_utilization_);
     }
-    if (frame_buffer_pool_ != nullptr)
-      frame_buffer_pool_->ReleaseBufferHold(buffer_id_);
+    buffer_read_permission_.reset();
     max_consumer_utilization_ =
         media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded;
   }
 }
 
 bool VideoCaptureController::BufferState::HasZeroConsumerHoldCount() {
   return consumer_hold_count_ == 0;
 }
 
+mojo::ScopedSharedBufferHandle
+VideoCaptureController::BufferState::CreateHandleCopy() {
+  return buffer_handle_->Clone();
+}
+
 void VideoCaptureController::BufferState::SetConsumerFeedbackObserver(
     media::VideoFrameConsumerFeedbackObserver* consumer_feedback_observer) {
   consumer_feedback_observer_ = consumer_feedback_observer;
 }
 
-void VideoCaptureController::BufferState::SetFrameBufferPool(
-    media::FrameBufferPool* frame_buffer_pool) {
-  frame_buffer_pool_ = frame_buffer_pool;
-}
-
 VideoCaptureController::VideoCaptureController()
-    : frame_buffer_pool_(nullptr),
-      consumer_feedback_observer_(nullptr),
+    : consumer_feedback_observer_(nullptr),
       state_(VIDEO_CAPTURE_STATE_STARTED),
       has_received_frames_(false),
       weak_ptr_factory_(this) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 }
 
 VideoCaptureController::~VideoCaptureController() = default;
 
 base::WeakPtr<VideoCaptureController>
 VideoCaptureController::GetWeakPtrForIOThread() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
-void VideoCaptureController::SetFrameBufferPool(
-    std::unique_ptr<media::FrameBufferPool> frame_buffer_pool) {
-  DCHECK_CURRENTLY_ON(BrowserThread::IO);
-  frame_buffer_pool_ = std::move(frame_buffer_pool);
-  // Update existing BufferState entries.
-  for (auto& entry : buffer_id_to_state_map_)
-    entry.second.SetFrameBufferPool(frame_buffer_pool_.get());
-}
-
 void VideoCaptureController::SetConsumerFeedbackObserver(
     std::unique_ptr<media::VideoFrameConsumerFeedbackObserver>
         consumer_feedback_observer) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   consumer_feedback_observer_ = std::move(consumer_feedback_observer);
   // Update existing BufferState entries.
-  for (auto& entry : buffer_id_to_state_map_)
-    entry.second.SetConsumerFeedbackObserver(consumer_feedback_observer_.get());
+  for (auto& entry : buffer_states_)
+    entry.SetConsumerFeedbackObserver(consumer_feedback_observer_.get());
 }
 
 void VideoCaptureController::AddClient(
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* event_handler,
     media::VideoCaptureSessionId session_id,
     const media::VideoCaptureParams& params) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DVLOG(1) << "VideoCaptureController::AddClient() -- id=" << id
            << ", session_id=" << session_id
@@ skipping 41 matching lines @@
 int VideoCaptureController::RemoveClient(
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* event_handler) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
   if (!client)
     return kInvalidMediaCaptureSessionId;
 
-  // Take back all buffers held by the |client|.
-  for (const auto& buffer_id : client->buffers_in_use)
-    buffer_id_to_state_map_.at(buffer_id).DecreaseConsumerCount();
+  for (const auto& buffer_id : client->buffers_in_use) {
+    OnClientFinishedConsumingBuffer(
+        client, buffer_id,
+        media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded);
+  }
   client->buffers_in_use.clear();
 
   int session_id = client->session_id;
   controller_clients_.remove_if(
       [client](const std::unique_ptr<ControllerClient>& ptr) {
         return ptr.get() == client;
       });
 
   return session_id;
 }
@@ skipping 83 matching lines @@
     NOTREACHED();
     return;
   }
   auto buffers_in_use_entry_iter =
       std::find(std::begin(client->buffers_in_use),
                 std::end(client->buffers_in_use), buffer_id);
   if (buffers_in_use_entry_iter == std::end(client->buffers_in_use)) {
     NOTREACHED();
     return;
   }
+  client->buffers_in_use.erase(buffers_in_use_entry_iter);
 
-  BufferState& buffer_state = buffer_id_to_state_map_.at(buffer_id);
-  buffer_state.RecordConsumerUtilization(consumer_resource_utilization);
-  buffer_state.DecreaseConsumerCount();
-  client->buffers_in_use.erase(buffers_in_use_entry_iter);
+  OnClientFinishedConsumingBuffer(client, buffer_id,
+                                  consumer_resource_utilization);
 }
 
 const media::VideoCaptureFormat&
 VideoCaptureController::GetVideoCaptureFormat() const {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return video_capture_format_;
 }
 
-void VideoCaptureController::OnIncomingCapturedVideoFrame(
-    media::VideoCaptureDevice::Client::Buffer buffer,
-    scoped_refptr<VideoFrame> frame) {
+void VideoCaptureController::OnNewBufferHandle(
+    int buffer_id,
+    std::unique_ptr<media::BufferHandleProvider> handle_provider) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
-  const int buffer_id = buffer.id();
+  DCHECK(std::find_if(buffer_states_.begin(), buffer_states_.end(),
+                      [buffer_id](const BufferState& entry) {
+                        return entry.buffer_id() == buffer_id;
+                      }) == buffer_states_.end());
+  buffer_states_.emplace_back(
+      buffer_id, consumer_feedback_observer_.get(),
+      handle_provider->GetHandleForInterProcessTransit());
+}
+
+void VideoCaptureController::OnFrameReadyInBuffer(
+    int buffer_id,
+    int frame_feedback_id,
+    std::unique_ptr<media::Ownership> buffer_read_permission,
+    media::mojom::VideoFrameInfoPtr frame_info) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK_NE(buffer_id, media::VideoCaptureBufferPool::kInvalidId);
 
-  // Insert if not exists.
-  const auto it =
-      buffer_id_to_state_map_
-          .insert(std::make_pair(
-              buffer_id, BufferState(buffer_id, buffer.frame_feedback_id(),
-                                     consumer_feedback_observer_.get(),
-                                     frame_buffer_pool_.get())))
-          .first;
-  BufferState& buffer_state = it->second;
-  DCHECK(buffer_state.HasZeroConsumerHoldCount());
+  auto buffer_state_iter =
+      std::find_if(buffer_states_.begin(), buffer_states_.end(),
+                   [buffer_id](const BufferState& entry) {
+                     return entry.buffer_id() == buffer_id;
+                   });
+  DCHECK(buffer_state_iter != buffer_states_.end());
+  DCHECK(buffer_state_iter->HasZeroConsumerHoldCount());
+  buffer_state_iter->set_frame_feedback_id(frame_feedback_id);
 
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
-    if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
-      frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
-                                   video_capture_format_.frame_rate);
-    }
-    std::unique_ptr<base::DictionaryValue> metadata =
-        frame->metadata()->CopyInternalValues();
-
-    // Only I420 and Y16 pixel formats are currently supported.
-    DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
-           frame->format() == media::PIXEL_FORMAT_Y16)
-        << "Unsupported pixel format: "
-        << media::VideoPixelFormatToString(frame->format());
-
-    // Sanity-checks to confirm |frame| is actually being backed by |buffer|.
-    auto buffer_access = buffer.handle_provider->GetHandleForInProcessAccess();
-    DCHECK(frame->storage_type() == media::VideoFrame::STORAGE_SHMEM);
-    DCHECK(frame->data(media::VideoFrame::kYPlane) >= buffer_access->data() &&
-           (frame->data(media::VideoFrame::kYPlane) <
-            (buffer_access->data() + buffer_access->mapped_size())))
-        << "VideoFrame does not appear to be backed by Buffer";
-
     for (const auto& client : controller_clients_) {
       if (client->session_closed || client->paused)
         continue;
 
-      // On the first use of a buffer on a client, share the memory handles.
+      // On the first use of a buffer on a client, call OnBufferCreated().
       auto known_buffers_entry_iter =
           std::find(std::begin(client->known_buffers),
                     std::end(client->known_buffers), buffer_id);
       bool is_new_buffer = false;
       if (known_buffers_entry_iter == std::end(client->known_buffers)) {
         client->known_buffers.push_back(buffer_id);
         is_new_buffer = true;
       }
       if (is_new_buffer) {
-        mojo::ScopedSharedBufferHandle handle =
-            buffer.handle_provider->GetHandleForInterProcessTransit();
+        size_t mapped_size =
+            media::VideoCaptureFormat(frame_info->coded_size, 0.0f,
+                                      frame_info->pixel_format,
+                                      frame_info->storage_type)
+                .ImageAllocationSize();
         client->event_handler->OnBufferCreated(
-            client->controller_id, std::move(handle),
-            buffer_access->mapped_size(), buffer_id);
+            client->controller_id, buffer_state_iter->CreateHandleCopy(),
+            mapped_size, buffer_id);
       }
+
+      auto frame_info_copy = CloneFrameInfo(&frame_info);
       client->event_handler->OnBufferReady(client->controller_id, buffer_id,
-                                           frame);
-
+                                           std::move(frame_info_copy));
       auto buffers_in_use_entry_iter =
           std::find(std::begin(client->buffers_in_use),
                     std::end(client->buffers_in_use), buffer_id);
       if (buffers_in_use_entry_iter == std::end(client->buffers_in_use))
         client->buffers_in_use.push_back(buffer_id);
       else
         DCHECK(false) << "Unexpected duplicate buffer: " << buffer_id;
-      buffer_state.IncreaseConsumerCount();
+      // Use if-check, because we are in a loop and only want to do this
+      // move operation once.
+      if (buffer_read_permission) {
+        buffer_state_iter->set_read_permission(
+            std::move(buffer_read_permission));
+      }
+      buffer_state_iter->IncreaseConsumerCount();
     }
   }
 
   if (!has_received_frames_) {
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
-                         frame->visible_rect().width());
+                         frame_info->coded_size.width());
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
-                         frame->visible_rect().height());
+                         frame_info->coded_size.height());
     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
-                               frame->visible_rect().width(),
-                               frame->visible_rect().height());
+                               frame_info->coded_size.width(),
+                               frame_info->coded_size.height());
     double frame_rate = 0.0f;
-    if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
-                                      &frame_rate)) {
+    media::VideoFrameMetadata metadata;
+    metadata.MergeInternalValuesFrom(*frame_info->metadata);
+    if (!metadata.GetDouble(VideoFrameMetadata::FRAME_RATE, &frame_rate)) {
       frame_rate = video_capture_format_.frame_rate;
     }
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
     has_received_frames_ = true;
   }
 }
 
+void VideoCaptureController::OnBufferRetired(int buffer_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  auto buffer_state_iter =
+      std::find_if(buffer_states_.begin(), buffer_states_.end(),
+                   [buffer_id](const BufferState& entry) {
+                     return entry.buffer_id() == buffer_id;
+                   });
+  DCHECK(buffer_state_iter != buffer_states_.end());
+
+  // If there are any clients still using the buffer, we need to allow them
+  // to finish up. We need to hold on to the BufferState entry until then,
+  // because it contains the |read_access_permission|.
+  if (buffer_state_iter->HasZeroConsumerHoldCount()) {
+    ReleaseBufferState(buffer_state_iter);
+  } else {
+    buffer_state_iter->set_is_retired();
+  }
+}
+
 void VideoCaptureController::OnError() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
        continue;
     client->event_handler->OnError(client->controller_id);
   }
 }
 
 void VideoCaptureController::OnLog(const std::string& message) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 }
 
-void VideoCaptureController::OnBufferDestroyed(int buffer_id_to_drop) {
-  DCHECK_CURRENTLY_ON(BrowserThread::IO);
-
-  for (const auto& client : controller_clients_) {
-    if (client->session_closed)
-      continue;
-
-    auto known_buffers_entry_iter =
-        std::find(std::begin(client->known_buffers),
-                  std::end(client->known_buffers), buffer_id_to_drop);
-    if (known_buffers_entry_iter != std::end(client->known_buffers)) {
-      client->known_buffers.erase(known_buffers_entry_iter);
-      client->event_handler->OnBufferDestroyed(client->controller_id,
-                                               buffer_id_to_drop);
-    }
-  }
-
-  buffer_id_to_state_map_.erase(buffer_id_to_drop);
-}
-
 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* handler,
     const ControllerClients& clients) {
   for (const auto& client : clients) {
     if (client->controller_id == id && client->event_handler == handler)
       return client.get();
   }
   return nullptr;
 }
 
 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
     int session_id,
     const ControllerClients& clients) {
   for (const auto& client : clients) {
     if (client->session_id == session_id)
       return client.get();
   }
   return nullptr;
 }
 
+void VideoCaptureController::OnClientFinishedConsumingBuffer(
+    ControllerClient* client,
+    int buffer_id,
+    double consumer_resource_utilization) {
+  auto buffer_state_iter =
+      std::find_if(buffer_states_.begin(), buffer_states_.end(),
+                   [buffer_id](const BufferState& entry) {
+                     return entry.buffer_id() == buffer_id;
+                   });
+  DCHECK(buffer_state_iter != buffer_states_.end());
+
+  buffer_state_iter->RecordConsumerUtilization(consumer_resource_utilization);
+  buffer_state_iter->DecreaseConsumerCount();
+  if (buffer_state_iter->HasZeroConsumerHoldCount() &&
+      buffer_state_iter->is_retired()) {
+    ReleaseBufferState(buffer_state_iter);
+  }
+}
+
+void VideoCaptureController::ReleaseBufferState(
+    const std::vector<BufferState>::iterator& buffer_state_iter) {
+  for (const auto& client : controller_clients_) {
+    if (client->session_closed)
+      continue;
+
+    auto known_buffers_entry_iter = std::find(std::begin(client->known_buffers),
+                                              std::end(client->known_buffers),
+                                              buffer_state_iter->buffer_id());
+    if (known_buffers_entry_iter != std::end(client->known_buffers)) {
+      client->known_buffers.erase(known_buffers_entry_iter);
+      client->event_handler->OnBufferDestroyed(client->controller_id,
+                                               buffer_state_iter->buffer_id());
+    }
+  }
+  buffer_states_.erase(buffer_state_iter);
+}
+
 }  // namespace content