[Mojo Video Capture] Replace RESOURCE_UTILIZATION with interface ReceiverLoadObserver

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
@@ skipping 28 matching lines @@
 
 namespace {
 
 static const int kInfiniteRatio = 99999;
 
 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
     UMA_HISTOGRAM_SPARSE_SLOWLY( \
         name, \
         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
 
-class SyncTokenClientImpl : public VideoFrame::SyncTokenClient {
- public:
-  explicit SyncTokenClientImpl(display_compositor::GLHelper* gl_helper)
-      : gl_helper_(gl_helper) {}
-  ~SyncTokenClientImpl() override {}
-  void GenerateSyncToken(gpu::SyncToken* sync_token) override {
-    gl_helper_->GenerateSyncToken(sync_token);
-  }
-  void WaitSyncToken(const gpu::SyncToken& sync_token) override {
-    gl_helper_->WaitSyncToken(sync_token);
-  }
-
- private:
-  display_compositor::GLHelper* gl_helper_;
-};
-
-void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
-                      const gpu::SyncToken& sync_token) {
-  DCHECK_CURRENTLY_ON(BrowserThread::UI);
-#if defined(OS_ANDROID)
-  NOTREACHED();
-#else
-  display_compositor::GLHelper* gl_helper =
-      ImageTransportFactory::GetInstance()->GetGLHelper();
-  // UpdateReleaseSyncToken() creates a new sync_token using |gl_helper|, so
-  // wait the given |sync_token| using |gl_helper|.
-  if (gl_helper) {
-    gl_helper->WaitSyncToken(sync_token);
-    SyncTokenClientImpl client(gl_helper);
-    video_frame->UpdateReleaseSyncToken(&client);
-  }
-#endif
-}
-
 std::unique_ptr<media::VideoCaptureJpegDecoder> CreateGpuJpegDecoder(
     const media::VideoCaptureJpegDecoder::DecodeDoneCB& decode_done_cb) {
   return base::MakeUnique<VideoCaptureGpuJpegDecoder>(decode_done_cb);
 }
 
 // Decorator for media::VideoFrameReceiver that forwards all incoming calls
 // to the Browser IO thread.
 class VideoFrameReceiverOnIOThread : public media::VideoFrameReceiver {
  public:
   explicit VideoFrameReceiverOnIOThread(
@@ skipping 50 matching lines @@
   // ID used for identifying this object.
   const VideoCaptureControllerID controller_id;
   VideoCaptureControllerEventHandler* const event_handler;
 
   const media::VideoCaptureSessionId session_id;
   const media::VideoCaptureParams parameters;
 
   // Buffers that are currently known to this client.
   std::set<int> known_buffers;
 
-  // Buffers currently held by this client, and sync token callback to call when
-  // they are returned from the client.
-  typedef std::map<int, scoped_refptr<VideoFrame>> ActiveBufferMap;
-  ActiveBufferMap active_buffers;
+  // Buffers currently held by this client.
+  std::set<int> buffers_in_use;

[miu, 2016/12/02 23:29:32]

Note on use of std::set/map/list/unordered_map: These are actually very
inefficient data structures when containing small numbers (less than a hundred
or so) of small-sized (16 bytes or less) elements. There's a "Using vector based
sets/maps to improve performance" thread on chromium-dev@ right now, and past
threads on that mailing list and the project-trim@ list about this.

Even when performance or efficiency does not seem to be an issue, use of these
data structures tends to "bust" the CPU cache, which causes other, unrelated
"hot" code paths to take a unexpected performance penalty.

The new guidance is to use std::vector for storage, and to do linear scans
before inserting (use std::find(vect.begin(), vect.end(), value)) if avoiding an
insert of duplicate entries is required. This will be more CPU and memory
efficient on modern architectures. Someone is looking into adding a "flat map"
in src/base, which will provide the std::map API, but be backed by the
more-efficient std::vector.

So, in new code (and if you prefer, fix old code as you come across it), please
try to use std::vector for small-sized values and a small numbers of elements.

[chfremer, 2016/12/03 00:30:35]

Done.

 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
   // client is sent an OnEnded() event. However, because the client retains a
   // VideoCaptureController* pointer, its ControllerClient entry lives on until
   // it unregisters itself via RemoveClient(), which may happen asynchronously.
   //
   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
   // OnEnded() events were processed synchronously (with the RemoveClient() done
   // implicitly), we could avoid tracking this state here in the Controller, and
   // simplify the code in both places.
   bool session_closed;
 
   // Indicates whether the client is paused, if true, VideoCaptureController
   // stops updating its buffer.
   bool paused;
 };
 
+VideoCaptureController::BufferState::BufferState(
+    int buffer_id,
+    int frame_feedback_id,
+    media::ConsumerFeedbackObserver* consumer_feedback_observer,
+    const scoped_refptr<media::VideoCaptureBufferPool>& buffer_pool,

[miu, 2016/12/02 23:29:32]

New smart pointer style guidelines: Both the |buffer_pool| and |frame| arguments
should not be passed by const-ref. Instead:

  scoped_refptr<media::VideoCaptureBufferPool> buffer_pool,
  scoped_refptr<media::VideoFrame> frame

...and then use std::move(x) in the initializer list to avoid redundant inc/dec
of the ref-counters.

[chfremer, 2016/12/03 00:30:35]

Done.

+    const scoped_refptr<media::VideoFrame>& frame)
+    : buffer_id_(buffer_id),
+      frame_feedback_id_(frame_feedback_id),
+      consumer_feedback_observer_(consumer_feedback_observer),
+      buffer_pool_(buffer_pool),
+      frame_(frame),
+      max_consumer_utilization_(
+          media::ConsumerFeedbackObserver::kNoUtilizationRecorded),
+      consumer_hold_count_(0) {}
+
+VideoCaptureController::BufferState::~BufferState() = default;
+
+VideoCaptureController::BufferState::BufferState(
+    const VideoCaptureController::BufferState& other) = default;
+
+void VideoCaptureController::BufferState::RecordConsumerUtilization(
+    double utilization) {
+  if (std::isfinite(utilization) && utilization >= 0.0) {
+    max_consumer_utilization_ =
+        std::max(max_consumer_utilization_, utilization);
+  }
+}
+
+void VideoCaptureController::BufferState::IncreaseConsumerCount() {
+  if (consumer_hold_count_ == 0)
+    buffer_pool_->HoldForConsumers(buffer_id_, 1);
+  consumer_hold_count_++;
+}
+
+void VideoCaptureController::BufferState::DecreaseConsumerCount() {
+  consumer_hold_count_--;
+  if (consumer_hold_count_ == 0) {
+    if (consumer_feedback_observer_ != nullptr &&
+        max_consumer_utilization_ !=
+            media::ConsumerFeedbackObserver::kNoUtilizationRecorded) {
+      consumer_feedback_observer_->OnConsumerReportingUtilization(
+          frame_feedback_id_, max_consumer_utilization_);
+    }
+    buffer_pool_->RelinquishConsumerHold(buffer_id_, 1);
+    max_consumer_utilization_ =
+        media::ConsumerFeedbackObserver::kNoUtilizationRecorded;
+  }
+}
+
+bool VideoCaptureController::BufferState::HasZeroConsumerHoldCount() {
+  return consumer_hold_count_ == 0;
+}
+
+void VideoCaptureController::BufferState::SetConsumerFeedbackObserver(
+    media::ConsumerFeedbackObserver* consumer_feedback_observer) {
+  consumer_feedback_observer_ = consumer_feedback_observer;
+}
+
 VideoCaptureController::VideoCaptureController(int max_buffers)
     : buffer_pool_(new media::VideoCaptureBufferPoolImpl(
           base::MakeUnique<media::VideoCaptureBufferTrackerFactoryImpl>(),
           max_buffers)),
+      consumer_feedback_observer_(nullptr),
       state_(VIDEO_CAPTURE_STATE_STARTED),
       has_received_frames_(false),
       weak_ptr_factory_(this) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 }
 
 base::WeakPtr<VideoCaptureController>
 VideoCaptureController::GetWeakPtrForIOThread() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
+void VideoCaptureController::SetConsumerFeedbackObserver(
+    std::unique_ptr<media::ConsumerFeedbackObserver>
+        consumer_feedback_observer) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  consumer_feedback_observer_ = std::move(consumer_feedback_observer);
+  // Update existing BufferState entries.
+  for (auto& entry : buffer_id_to_state_map_)
+    entry.second.SetConsumerFeedbackObserver(consumer_feedback_observer_.get());
+}
+
 std::unique_ptr<media::VideoCaptureDevice::Client>
 VideoCaptureController::NewDeviceClient() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return base::MakeUnique<media::VideoCaptureDeviceClient>(
       base::MakeUnique<VideoFrameReceiverOnIOThread>(
           this->GetWeakPtrForIOThread()),
       buffer_pool_,
       base::Bind(&CreateGpuJpegDecoder,
                  base::Bind(&VideoFrameReceiver::OnIncomingCapturedVideoFrame,
                             this->GetWeakPtrForIOThread())));
@@ skipping 52 matching lines @@
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* event_handler) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
   if (!client)
     return kInvalidMediaCaptureSessionId;
 
   // Take back all buffers held by the |client|.
-  for (const auto& buffer : client->active_buffers)
-    buffer_pool_->RelinquishConsumerHold(buffer.first, 1);
-  client->active_buffers.clear();
+  for (const auto& buffer_id : client->buffers_in_use)
+    buffer_id_to_state_map_.at(buffer_id).DecreaseConsumerCount();
+  client->buffers_in_use.clear();
 
   int session_id = client->session_id;
   controller_clients_.remove_if(
       [client](const std::unique_ptr<ControllerClient>& ptr) {
         return ptr.get() == client;
       });
 
   return session_id;
 }
 
@@ skipping 71 matching lines @@
     VideoCaptureControllerEventHandler* event_handler,
     int buffer_id,
     const gpu::SyncToken& sync_token,
     double consumer_resource_utilization) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
-  ControllerClient::ActiveBufferMap::iterator iter;
-  if (!client || (iter = client->active_buffers.find(buffer_id)) ==
-                     client->active_buffers.end()) {
+  if (!client || (client->buffers_in_use.find(buffer_id) ==
+                  client->buffers_in_use.end())) {
     NOTREACHED();
     return;
   }
 
-  // Set the RESOURCE_UTILIZATION to the maximum of those provided by each
-  // consumer (via separate calls to this method that refer to the same
-  // VideoFrame).  The producer of this VideoFrame may check this value, after
-  // all consumer holds are relinquished, to make quality versus performance
-  // trade-off decisions.
-  scoped_refptr<VideoFrame> frame = iter->second;
-  if (std::isfinite(consumer_resource_utilization) &&
-      consumer_resource_utilization >= 0.0) {
-    double resource_utilization = -1.0;
-    if (frame->metadata()->GetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                     &resource_utilization)) {
-      frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                   std::max(consumer_resource_utilization,
-                                            resource_utilization));
-    } else {
-      frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                   consumer_resource_utilization);
-    }
-  }
-
-  client->active_buffers.erase(iter);
-  buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
-
-#if defined(OS_ANDROID)
-  DCHECK(!sync_token.HasData());
-#endif
-  if (sync_token.HasData())
-    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
-                            base::Bind(&ReturnVideoFrame, frame, sync_token));
+  BufferState& buffer_state = buffer_id_to_state_map_.at(buffer_id);
+  buffer_state.RecordConsumerUtilization(consumer_resource_utilization);
+  buffer_state.DecreaseConsumerCount();
+  client->buffers_in_use.erase(buffer_id);
 }
 
 const media::VideoCaptureFormat&
 VideoCaptureController::GetVideoCaptureFormat() const {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return video_capture_format_;
 }
 
 VideoCaptureController::~VideoCaptureController() {
 }
 
 void VideoCaptureController::OnIncomingCapturedVideoFrame(
     std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer,
     scoped_refptr<VideoFrame> frame) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   const int buffer_id = buffer->id();
   DCHECK_NE(buffer_id, media::VideoCaptureBufferPool::kInvalidId);
 
-  int count = 0;
+  // Insert if not exists.
+  const auto it =
+      buffer_id_to_state_map_
+          .insert(std::make_pair(
+              buffer_id, BufferState(buffer_id, buffer->frame_feedback_id(),
+                                     consumer_feedback_observer_.get(),
+                                     buffer_pool_, frame)))
+          .first;
+  BufferState& buffer_state = it->second;
+  DCHECK(buffer_state.HasZeroConsumerHoldCount());
+
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
       frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
                                    video_capture_format_.frame_rate);
     }
     std::unique_ptr<base::DictionaryValue> metadata(
         new base::DictionaryValue());
     frame->metadata()->MergeInternalValuesInto(metadata.get());
 
     // Only I420 and Y16 pixel formats are currently supported.
@@ skipping 14 matching lines @@
       if (client->session_closed || client->paused)
         continue;
 
       // On the first use of a buffer on a client, share the memory handles.
       const bool is_new_buffer = client->known_buffers.insert(buffer_id).second;
       if (is_new_buffer)
         DoNewBufferOnIOThread(client.get(), buffer.get(), frame);
 
       client->event_handler->OnBufferReady(client->controller_id, buffer_id,
                                            frame);
-      const bool inserted =
-          client->active_buffers.insert(std::make_pair(buffer_id, frame))
-              .second;
+      const bool inserted = client->buffers_in_use.insert(buffer_id).second;
       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer_id;
-      count++;
+      buffer_state.IncreaseConsumerCount();
     }
   }
 
   if (!has_received_frames_) {
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
                          frame->visible_rect().width());
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
                          frame->visible_rect().height());
     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
                                frame->visible_rect().width(),
                                frame->visible_rect().height());
     double frame_rate = 0.0f;
     if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
                                       &frame_rate)) {
       frame_rate = video_capture_format_.frame_rate;
     }
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
     has_received_frames_ = true;
   }
-
-  buffer_pool_->HoldForConsumers(buffer_id, count);
 }
 
 void VideoCaptureController::OnError() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
        continue;
     client->event_handler->OnError(client->controller_id);
@@ skipping 10 matching lines @@
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
       continue;
 
     if (client->known_buffers.erase(buffer_id_to_drop)) {
       client->event_handler->OnBufferDestroyed(client->controller_id,
                                                buffer_id_to_drop);
     }
   }
+
+  buffer_id_to_state_map_.erase(buffer_id_to_drop);
 }
 
 void VideoCaptureController::DoNewBufferOnIOThread(
     ControllerClient* client,
     media::VideoCaptureDevice::Client::Buffer* buffer,
     const scoped_refptr<media::VideoFrame>& frame) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK_EQ(media::VideoFrame::STORAGE_SHMEM, frame->storage_type());
 
   const int buffer_id = buffer->id();
@@ skipping 19 matching lines @@
     int session_id,
     const ControllerClients& clients) {
   for (const auto& client : clients) {
     if (client->session_id == session_id)
       return client.get();
   }
   return nullptr;
 }
 
 }  // namespace content