[Mojo Video Capture] Split OnIncomingCapturedVideoFrame() to OnNewBuffer() and OnFrameReadyInBuffer(

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
 #include <set>
 
 #include "base/bind.h"
 #include "base/command_line.h"
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "build/build_config.h"
 #include "components/display_compositor/gl_helper.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
+#include "content/common/video_capture.mojom.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/common/content_switches.h"
 #include "media/base/video_frame.h"
 #include "media/capture/video/video_capture_buffer_pool.h"
 #include "media/capture/video/video_capture_buffer_tracker_factory_impl.h"
 #include "media/capture/video/video_capture_device_client.h"
 #include "mojo/public/cpp/system/platform_handle.h"
 
 #if !defined(OS_ANDROID)
 #include "content/browser/compositor/image_transport_factory.h"
 #endif
 
 using media::VideoCaptureFormat;
 using media::VideoFrame;
 using media::VideoFrameMetadata;
 
 namespace content {
 
 namespace {
 
 static const int kInfiniteRatio = 99999;
 
 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
   UMA_HISTOGRAM_SPARSE_SLOWLY(                          \
       name, (height) ? ((width)*100) / (height) : kInfiniteRatio);
 
+// This manual copy routine is needed because base::DictionaryValue does not

[miu, 2017/02/14 00:41:27]

Maybe it'd be better to fix the mojo impl? Meaning, the copy ctor generated by
mojo should handle dictionary values correctly, because we use those all over
the Chromium project. Not sure if this means fixing typemaps, or something
internal to the mojo code generator.

The danger for leaving this as-is, is that someone could add fields to
media::mojom::VideoFrameInfo, and not account for them here. You could, I
suppose, add a DCHECK that does an "equals" comparison between |frame_info| and
|result|.

[chfremer, 2017/02/14 21:15:16]

Excellent suggestion. I didn't fully realize how dangerous this copy routine
was.
I had a look at the Mojo code, and found that it should be relatively simple to 
support cloning DictionaryValue by adding a specialization for it to template 
CloneTraits. Added to next patch set and adding rockot@ to check if that is 
a valid approach.

+// allow the default copy operator.
+media::mojom::VideoFrameInfoPtr CloneFrameInfo(
+    media::mojom::VideoFrameInfoPtr* frame_info) {

[miu, 2017/02/14 00:41:27]

Seems that the type of this argument should be:

  const media::mojom::VideoFrameInfo& frame_info

[chfremer, 2017/02/14 21:15:16]

Done.

+  auto result = media::mojom::VideoFrameInfo::New();
+  result->timestamp = (*frame_info)->timestamp;
+  result->pixel_format = (*frame_info)->pixel_format;
+  result->storage_type = (*frame_info)->storage_type;
+  result->coded_size = (*frame_info)->coded_size;
+  result->visible_rect = (*frame_info)->visible_rect;
+  result->metadata = base::MakeUnique<base::DictionaryValue>();
+  result->metadata->MergeDictionary((*frame_info)->metadata.get());
+  return result;
+}
+
 }  // anonymous namespace
 
 struct VideoCaptureController::ControllerClient {
   ControllerClient(VideoCaptureControllerID id,
                    VideoCaptureControllerEventHandler* handler,
                    media::VideoCaptureSessionId session_id,
                    const media::VideoCaptureParams& params)
       : controller_id(id),
         event_handler(handler),
         session_id(session_id),
@@ skipping 28 matching lines @@
 
   // Indicates whether the client is paused, if true, VideoCaptureController
   // stops updating its buffer.
   bool paused;
 };
 
 VideoCaptureController::BufferContext::BufferContext(
     int buffer_context_id,
     int buffer_id,
     media::VideoFrameConsumerFeedbackObserver* consumer_feedback_observer,
-    media::FrameBufferPool* frame_buffer_pool)
+    mojo::ScopedSharedBufferHandle handle)
     : buffer_context_id_(buffer_context_id),
       buffer_id_(buffer_id),
       is_retired_(false),
       frame_feedback_id_(0),
       consumer_feedback_observer_(consumer_feedback_observer),
-      frame_buffer_pool_(frame_buffer_pool),
+      buffer_handle_(std::move(handle)),
       max_consumer_utilization_(
           media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded),
       consumer_hold_count_(0) {}
 
 VideoCaptureController::BufferContext::~BufferContext() = default;
 
 VideoCaptureController::BufferContext::BufferContext(
-    const VideoCaptureController::BufferContext& other) = default;
+    VideoCaptureController::BufferContext&& other) = default;
 
 VideoCaptureController::BufferContext& VideoCaptureController::BufferContext::
-operator=(const BufferContext& other) = default;
+operator=(BufferContext&& other) = default;
 
 void VideoCaptureController::BufferContext::RecordConsumerUtilization(
     double utilization) {
   if (std::isfinite(utilization) && utilization >= 0.0) {
     max_consumer_utilization_ =
         std::max(max_consumer_utilization_, utilization);
   }
 }
 
 void VideoCaptureController::BufferContext::IncreaseConsumerCount() {
-  if (consumer_hold_count_ == 0)
-    if (frame_buffer_pool_ != nullptr)
-      frame_buffer_pool_->SetBufferHold(buffer_id_);
   consumer_hold_count_++;
 }
 
 void VideoCaptureController::BufferContext::DecreaseConsumerCount() {
   consumer_hold_count_--;
   if (consumer_hold_count_ == 0) {
     if (consumer_feedback_observer_ != nullptr &&
         max_consumer_utilization_ !=
             media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded) {
       consumer_feedback_observer_->OnUtilizationReport(
           frame_feedback_id_, max_consumer_utilization_);
     }
-    if (frame_buffer_pool_ != nullptr)
-      frame_buffer_pool_->ReleaseBufferHold(buffer_id_);
+    buffer_read_permission_.reset();
     max_consumer_utilization_ =
         media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded;
   }
 }
 
 bool VideoCaptureController::BufferContext::HasZeroConsumerHoldCount() {
   return consumer_hold_count_ == 0;
 }
 
+mojo::ScopedSharedBufferHandle
+VideoCaptureController::BufferContext::CreateHandleCopy() {
+  return buffer_handle_->Clone();
+}
+
 VideoCaptureController::VideoCaptureController()
-    : frame_buffer_pool_(nullptr),
-      consumer_feedback_observer_(nullptr),
+    : consumer_feedback_observer_(nullptr),
       state_(VIDEO_CAPTURE_STATE_STARTED),
       has_received_frames_(false),
       weak_ptr_factory_(this) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 }
 
 VideoCaptureController::~VideoCaptureController() = default;
 
 base::WeakPtr<VideoCaptureController>
 VideoCaptureController::GetWeakPtrForIOThread() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
-void VideoCaptureController::SetFrameBufferPool(
-    std::unique_ptr<media::FrameBufferPool> frame_buffer_pool) {
-  DCHECK_CURRENTLY_ON(BrowserThread::IO);
-  frame_buffer_pool_ = std::move(frame_buffer_pool);
-  // Update existing BufferContext entries.
-  for (auto& entry : buffer_contexts_)
-    entry.set_frame_buffer_pool(frame_buffer_pool_.get());
-}
-
 void VideoCaptureController::SetConsumerFeedbackObserver(
     std::unique_ptr<media::VideoFrameConsumerFeedbackObserver>
         consumer_feedback_observer) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   consumer_feedback_observer_ = std::move(consumer_feedback_observer);
   // Update existing BufferContext entries.
   for (auto& entry : buffer_contexts_)
     entry.set_consumer_feedback_observer(consumer_feedback_observer_.get());
 }
 
@@ skipping 167 matching lines @@
   OnClientFinishedConsumingBuffer(client, buffer_id,
                                   consumer_resource_utilization);
 }
 
 const media::VideoCaptureFormat& VideoCaptureController::GetVideoCaptureFormat()
     const {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return video_capture_format_;
 }
 
-void VideoCaptureController::OnIncomingCapturedVideoFrame(
-    media::VideoCaptureDevice::Client::Buffer buffer,
-    scoped_refptr<VideoFrame> frame) {
+void VideoCaptureController::OnNewBufferHandle(
+    int buffer_id,
+    std::unique_ptr<media::VideoCaptureDevice::Client::Buffer::HandleProvider>
+        handle_provider) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
-  const int buffer_id_from_producer = buffer.id();
-  DCHECK_NE(buffer_id_from_producer, media::VideoCaptureBufferPool::kInvalidId);
-  auto buffer_context_iter =
-      FindUnretiredBufferContextFromBufferId(buffer_id_from_producer);
-  if (buffer_context_iter == buffer_contexts_.end()) {
-    // A new buffer has been shared with us. Create new BufferContext.
-    buffer_contexts_.emplace_back(
-        next_buffer_context_id_++, buffer_id_from_producer,
-        consumer_feedback_observer_.get(), frame_buffer_pool_.get());
-    buffer_context_iter = buffer_contexts_.end() - 1;
-  }
-  buffer_context_iter->set_frame_feedback_id(buffer.frame_feedback_id());
+  DCHECK(FindUnretiredBufferContextFromBufferId(buffer_id) ==
+         buffer_contexts_.end());
+  buffer_contexts_.emplace_back(
+      next_buffer_context_id_++, buffer_id, consumer_feedback_observer_.get(),
+      handle_provider->GetHandleForInterProcessTransit());
+}
+
+void VideoCaptureController::OnFrameReadyInBuffer(
+    int buffer_id,
+    int frame_feedback_id,
+    std::unique_ptr<
+        media::VideoCaptureDevice::Client::Buffer::ScopedAccessPermission>
+        buffer_read_permission,
+    media::mojom::VideoFrameInfoPtr frame_info) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK_NE(buffer_id, media::VideoCaptureBufferPool::kInvalidId);
+
+  auto buffer_context_iter = FindUnretiredBufferContextFromBufferId(buffer_id);
+  DCHECK(buffer_context_iter != buffer_contexts_.end());
+  buffer_context_iter->set_frame_feedback_id(frame_feedback_id);
   DCHECK(buffer_context_iter->HasZeroConsumerHoldCount());
 
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
-    if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
-      frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
-                                   video_capture_format_.frame_rate);
-    }
-    std::unique_ptr<base::DictionaryValue> metadata =
-        frame->metadata()->CopyInternalValues();
-
-    // Only I420 and Y16 pixel formats are currently supported.
-    DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
-           frame->format() == media::PIXEL_FORMAT_Y16)
-        << "Unsupported pixel format: "
-        << media::VideoPixelFormatToString(frame->format());
-
-    // Sanity-checks to confirm |frame| is actually being backed by |buffer|.
-    auto buffer_access =
-        buffer.handle_provider()->GetHandleForInProcessAccess();
-    DCHECK(frame->storage_type() == media::VideoFrame::STORAGE_SHMEM);
-    DCHECK(frame->data(media::VideoFrame::kYPlane) >= buffer_access->data() &&
-           (frame->data(media::VideoFrame::kYPlane) <
-            (buffer_access->data() + buffer_access->mapped_size())))
-        << "VideoFrame does not appear to be backed by Buffer";
-
     const int buffer_context_id = buffer_context_iter->buffer_context_id();
     for (const auto& client : controller_clients_) {
       if (client->session_closed || client->paused)
         continue;
 
-      // On the first use of a BufferContext on a client, share the memory
+      // On the first use of a BufferContext on a client, call
+      // OnBufferCreated().
       // handles.
       auto known_buffers_entry_iter = std::find(
           std::begin(client->known_buffer_context_ids),
           std::end(client->known_buffer_context_ids), buffer_context_id);
       bool is_new_buffer = false;
       if (known_buffers_entry_iter ==
           std::end(client->known_buffer_context_ids)) {
         client->known_buffer_context_ids.push_back(buffer_context_id);
         is_new_buffer = true;
       }
       if (is_new_buffer) {
-        mojo::ScopedSharedBufferHandle handle =
-            buffer.handle_provider()->GetHandleForInterProcessTransit();
+        size_t mapped_size =
+            media::VideoCaptureFormat(frame_info->coded_size, 0.0f,
+                                      frame_info->pixel_format,
+                                      frame_info->storage_type)
+                .ImageAllocationSize();
         client->event_handler->OnBufferCreated(
-            client->controller_id, std::move(handle),
-            buffer_access->mapped_size(), buffer_context_id);
+            client->controller_id, buffer_context_iter->CreateHandleCopy(),
+            mapped_size, buffer_context_id);
       }
-      client->event_handler->OnBufferReady(client->controller_id,
-                                           buffer_context_id, frame);
+
+      auto frame_info_copy = CloneFrameInfo(&frame_info);

[miu, 2017/02/14 00:41:27]

nit: You could just do:

  client->event_handler->OnBufferReady(
      client->controller_id, buffer_context_id,
      CloneFrameInfo(&frame_info));

[chfremer, 2017/02/14 21:15:16]

Done.

+      client->event_handler->OnBufferReady(
+          client->controller_id, buffer_context_id, std::move(frame_info_copy));
 
       auto buffers_in_use_entry_iter =
           std::find(std::begin(client->buffers_in_use),
                     std::end(client->buffers_in_use), buffer_context_id);
       if (buffers_in_use_entry_iter == std::end(client->buffers_in_use))
         client->buffers_in_use.push_back(buffer_context_id);
       else
         DCHECK(false) << "Unexpected duplicate buffer: " << buffer_context_id;
+
+      // Use if-check, because we are in a loop and only want to do this

[miu, 2017/02/14 00:41:27]

style nit: Don't use 2nd person in comments (i.e., "we" is not the focus).

[chfremer, 2017/02/14 21:15:16]

Done.

+      // move operation once.
+      if (buffer_read_permission) {
+        buffer_context_iter->set_read_permission(

[miu, 2017/02/14 00:41:27]

Perhaps this could be simplified by just calling set_read_permission() once
after the loop. Something like:

    for (...) {
      ...
      buffer_context_iter->IncreaseConsumerCount();
    }

    if (buffer_context_iter->HasConsumers()) {
      buffer_context_iter->set_read_permission(
          std::move(buffer_read_permission));
    }
  }

...or, if it's perfectly acceptable to always transfer the read permission (even
if it turns out there are no consumers), you could call set_read_permission()
unconditionally, either before or after the loop.

[chfremer, 2017/02/14 21:15:16]

Thanks. Moving the check to after the loop makes it much more clear and
eliminates 
the need for the comment trying to explain it :-).

+            std::move(buffer_read_permission));
+      }
       buffer_context_iter->IncreaseConsumerCount();
     }
   }
 
   if (!has_received_frames_) {
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
-                         frame->visible_rect().width());
+                         frame_info->coded_size.width());
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
-                         frame->visible_rect().height());
+                         frame_info->coded_size.height());
     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
-                               frame->visible_rect().width(),
-                               frame->visible_rect().height());
+                               frame_info->coded_size.width(),
+                               frame_info->coded_size.height());
     double frame_rate = 0.0f;
-    if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
-                                      &frame_rate)) {
+    media::VideoFrameMetadata metadata;
+    metadata.MergeInternalValuesFrom(*frame_info->metadata);
+    if (!metadata.GetDouble(VideoFrameMetadata::FRAME_RATE, &frame_rate)) {
       frame_rate = video_capture_format_.frame_rate;
     }
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
     has_received_frames_ = true;
   }
 }
 
+void VideoCaptureController::OnBufferRetired(int buffer_id) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  auto buffer_context_iter = FindUnretiredBufferContextFromBufferId(buffer_id);
+  DCHECK(buffer_context_iter != buffer_contexts_.end());
+
+  // If there are any clients still using the buffer, we need to allow them
+  // to finish up. We need to hold on to the BufferContext entry until then,
+  // because it contains the consumer hold.
+  if (buffer_context_iter->HasZeroConsumerHoldCount())
+    ReleaseBufferContext(buffer_context_iter);
+  else
+    buffer_context_iter->set_is_retired();
+}
+
 void VideoCaptureController::OnError() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
       continue;
     client->event_handler->OnError(client->controller_id);
   }
 }
 
 void VideoCaptureController::OnLog(const std::string& message) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 }
 
-void VideoCaptureController::OnBufferRetired(int buffer_id) {
-  DCHECK_CURRENTLY_ON(BrowserThread::IO);
-
-  auto buffer_context_iter = FindUnretiredBufferContextFromBufferId(buffer_id);
-  DCHECK(buffer_context_iter != buffer_contexts_.end());
-
-  // If there are any clients still using the buffer, we need to allow them
-  // to finish up. We need to hold on to the BufferContext entry until then,
-  // because it contains the consumer hold.
-  if (buffer_context_iter->HasZeroConsumerHoldCount())
-    ReleaseBufferContext(buffer_context_iter);
-  else
-    buffer_context_iter->set_is_retired();
-}
-
 VideoCaptureController::ControllerClient* VideoCaptureController::FindClient(
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* handler,
     const ControllerClients& clients) {
   for (const auto& client : clients) {
     if (client->controller_id == id && client->event_handler == handler)
       return client.get();
   }
   return nullptr;
 }
@@ skipping 53 matching lines @@
     if (entry_iter != std::end(client->known_buffer_context_ids)) {
       client->known_buffer_context_ids.erase(entry_iter);
       client->event_handler->OnBufferDestroyed(
           client->controller_id, buffer_context_iter->buffer_context_id());
     }
   }
   buffer_contexts_.erase(buffer_context_iter);
 }
 
 }  // namespace content