[Mojo Video Capture] Replace RESOURCE_UTILIZATION with interface ReceiverLoadObserver

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
@@ skipping 28 matching lines @@
 
 namespace {
 
 static const int kInfiniteRatio = 99999;
 
 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
     UMA_HISTOGRAM_SPARSE_SLOWLY( \
         name, \
         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
 
-class SyncTokenClientImpl : public VideoFrame::SyncTokenClient {
- public:
-  explicit SyncTokenClientImpl(display_compositor::GLHelper* gl_helper)
-      : gl_helper_(gl_helper) {}
-  ~SyncTokenClientImpl() override {}
-  void GenerateSyncToken(gpu::SyncToken* sync_token) override {
-    gl_helper_->GenerateSyncToken(sync_token);
-  }
-  void WaitSyncToken(const gpu::SyncToken& sync_token) override {
-    gl_helper_->WaitSyncToken(sync_token);
-  }
-
- private:
-  display_compositor::GLHelper* gl_helper_;
-};
-
-void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
-                      const gpu::SyncToken& sync_token) {
-  DCHECK_CURRENTLY_ON(BrowserThread::UI);
-#if defined(OS_ANDROID)
-  NOTREACHED();
-#else
-  display_compositor::GLHelper* gl_helper =
-      ImageTransportFactory::GetInstance()->GetGLHelper();
-  // UpdateReleaseSyncToken() creates a new sync_token using |gl_helper|, so
-  // wait the given |sync_token| using |gl_helper|.
-  if (gl_helper) {
-    gl_helper->WaitSyncToken(sync_token);
-    SyncTokenClientImpl client(gl_helper);
-    video_frame->UpdateReleaseSyncToken(&client);
-  }
-#endif
-}
-
 std::unique_ptr<media::VideoCaptureJpegDecoder> CreateGpuJpegDecoder(
     const media::VideoCaptureJpegDecoder::DecodeDoneCB& decode_done_cb) {
   return base::MakeUnique<VideoCaptureGpuJpegDecoder>(decode_done_cb);
 }
 
 // Decorator for media::VideoFrameReceiver that forwards all incoming calls
 // to the Browser IO thread.
 class VideoFrameReceiverOnIOThread : public media::VideoFrameReceiver {
  public:
   explicit VideoFrameReceiverOnIOThread(
@@ skipping 48 matching lines @@
   ~ControllerClient() {}
 
   // ID used for identifying this object.
   const VideoCaptureControllerID controller_id;
   VideoCaptureControllerEventHandler* const event_handler;
 
   const media::VideoCaptureSessionId session_id;
   const media::VideoCaptureParams parameters;
 
   // Buffers that are currently known to this client.
-  std::set<int> known_buffers;
+  std::vector<int> known_buffers;
 
-  // Buffers currently held by this client, and sync token callback to call when
-  // they are returned from the client.
-  typedef std::map<int, scoped_refptr<VideoFrame>> ActiveBufferMap;
-  ActiveBufferMap active_buffers;
+  // Buffers currently held by this client.
+  std::vector<int> buffers_in_use;
 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
   // client is sent an OnEnded() event. However, because the client retains a
   // VideoCaptureController* pointer, its ControllerClient entry lives on until
   // it unregisters itself via RemoveClient(), which may happen asynchronously.
   //
   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
   // OnEnded() events were processed synchronously (with the RemoveClient() done
   // implicitly), we could avoid tracking this state here in the Controller, and
   // simplify the code in both places.
   bool session_closed;
 
   // Indicates whether the client is paused, if true, VideoCaptureController
   // stops updating its buffer.
   bool paused;
 };
 
+VideoCaptureController::BufferState::BufferState(
+    int buffer_id,
+    int frame_feedback_id,
+    media::VideoFrameConsumerFeedbackObserver* consumer_feedback_observer,
+    scoped_refptr<media::VideoCaptureBufferPool> buffer_pool,
+    scoped_refptr<media::VideoFrame> frame)
+    : buffer_id_(buffer_id),
+      frame_feedback_id_(frame_feedback_id),
+      consumer_feedback_observer_(consumer_feedback_observer),
+      buffer_pool_(std::move(buffer_pool)),
+      frame_(std::move(frame)),
+      max_consumer_utilization_(
+          media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded),
+      consumer_hold_count_(0) {}
+
+VideoCaptureController::BufferState::~BufferState() = default;
+
+VideoCaptureController::BufferState::BufferState(
+    const VideoCaptureController::BufferState& other) = default;
+
+void VideoCaptureController::BufferState::RecordConsumerUtilization(
+    double utilization) {
+  if (std::isfinite(utilization) && utilization >= 0.0) {
+    max_consumer_utilization_ =
+        std::max(max_consumer_utilization_, utilization);
+  }
+}
+
+void VideoCaptureController::BufferState::IncreaseConsumerCount() {
+  if (consumer_hold_count_ == 0)
+    buffer_pool_->HoldForConsumers(buffer_id_, 1);
+  consumer_hold_count_++;
+}
+
+void VideoCaptureController::BufferState::DecreaseConsumerCount() {
+  consumer_hold_count_--;
+  if (consumer_hold_count_ == 0) {
+    if (consumer_feedback_observer_ != nullptr &&
+        max_consumer_utilization_ !=
+            media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded) {
+      consumer_feedback_observer_->OnUtilizationReport(
+          frame_feedback_id_, max_consumer_utilization_);
+    }
+    buffer_pool_->RelinquishConsumerHold(buffer_id_, 1);
+    max_consumer_utilization_ =
+        media::VideoFrameConsumerFeedbackObserver::kNoUtilizationRecorded;
+  }
+}
+
+bool VideoCaptureController::BufferState::HasZeroConsumerHoldCount() {
+  return consumer_hold_count_ == 0;
+}
+
+void VideoCaptureController::BufferState::SetConsumerFeedbackObserver(
+    media::VideoFrameConsumerFeedbackObserver* consumer_feedback_observer) {
+  consumer_feedback_observer_ = consumer_feedback_observer;
+}
+
 VideoCaptureController::VideoCaptureController(int max_buffers)
     : buffer_pool_(new media::VideoCaptureBufferPoolImpl(
           base::MakeUnique<media::VideoCaptureBufferTrackerFactoryImpl>(),
           max_buffers)),
+      consumer_feedback_observer_(nullptr),
       state_(VIDEO_CAPTURE_STATE_STARTED),
       has_received_frames_(false),
       weak_ptr_factory_(this) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 }
 
 base::WeakPtr<VideoCaptureController>
 VideoCaptureController::GetWeakPtrForIOThread() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
+void VideoCaptureController::SetConsumerFeedbackObserver(
+    std::unique_ptr<media::VideoFrameConsumerFeedbackObserver>
+        consumer_feedback_observer) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  consumer_feedback_observer_ = std::move(consumer_feedback_observer);
+  // Update existing BufferState entries.
+  for (auto& entry : buffer_id_to_state_map_)
+    entry.second.SetConsumerFeedbackObserver(consumer_feedback_observer_.get());
+}
+
 std::unique_ptr<media::VideoCaptureDevice::Client>
 VideoCaptureController::NewDeviceClient() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return base::MakeUnique<media::VideoCaptureDeviceClient>(
       base::MakeUnique<VideoFrameReceiverOnIOThread>(
           this->GetWeakPtrForIOThread()),
       buffer_pool_,
       base::Bind(&CreateGpuJpegDecoder,
                  base::Bind(&VideoFrameReceiver::OnIncomingCapturedVideoFrame,
                             this->GetWeakPtrForIOThread())));
@@ skipping 52 matching lines @@
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* event_handler) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
   if (!client)
     return kInvalidMediaCaptureSessionId;
 
   // Take back all buffers held by the |client|.
-  for (const auto& buffer : client->active_buffers)
-    buffer_pool_->RelinquishConsumerHold(buffer.first, 1);
-  client->active_buffers.clear();
+  for (const auto& buffer_id : client->buffers_in_use)
+    buffer_id_to_state_map_.at(buffer_id).DecreaseConsumerCount();
+  client->buffers_in_use.clear();
 
   int session_id = client->session_id;
   controller_clients_.remove_if(
       [client](const std::unique_ptr<ControllerClient>& ptr) {
         return ptr.get() == client;
       });
 
   return session_id;
 }
 
@@ skipping 71 matching lines @@
     VideoCaptureControllerEventHandler* event_handler,
     int buffer_id,
     const gpu::SyncToken& sync_token,
     double consumer_resource_utilization) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
-  ControllerClient::ActiveBufferMap::iterator iter;
-  if (!client || (iter = client->active_buffers.find(buffer_id)) ==
-                     client->active_buffers.end()) {
+  if (!client) {
+    NOTREACHED();
+    return;
+  }
+  auto buffers_in_use_entry_iter =
+      std::find(std::begin(client->buffers_in_use),
+                std::end(client->buffers_in_use), buffer_id);
+  if (buffers_in_use_entry_iter == std::end(client->buffers_in_use)) {
     NOTREACHED();
     return;
   }
 
-  // Set the RESOURCE_UTILIZATION to the maximum of those provided by each
-  // consumer (via separate calls to this method that refer to the same
-  // VideoFrame).  The producer of this VideoFrame may check this value, after
-  // all consumer holds are relinquished, to make quality versus performance
-  // trade-off decisions.
-  scoped_refptr<VideoFrame> frame = iter->second;
-  if (std::isfinite(consumer_resource_utilization) &&
-      consumer_resource_utilization >= 0.0) {
-    double resource_utilization = -1.0;
-    if (frame->metadata()->GetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                     &resource_utilization)) {
-      frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                   std::max(consumer_resource_utilization,
-                                            resource_utilization));
-    } else {
-      frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                   consumer_resource_utilization);
-    }
-  }
-
-  client->active_buffers.erase(iter);
-  buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
-
-#if defined(OS_ANDROID)
-  DCHECK(!sync_token.HasData());
-#endif
-  if (sync_token.HasData())
-    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
-                            base::Bind(&ReturnVideoFrame, frame, sync_token));
+  BufferState& buffer_state = buffer_id_to_state_map_.at(buffer_id);
+  buffer_state.RecordConsumerUtilization(consumer_resource_utilization);
+  buffer_state.DecreaseConsumerCount();
+  client->buffers_in_use.erase(buffers_in_use_entry_iter);
 }
 
 const media::VideoCaptureFormat&
 VideoCaptureController::GetVideoCaptureFormat() const {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return video_capture_format_;
 }
 
 VideoCaptureController::~VideoCaptureController() {
 }
 
 void VideoCaptureController::OnIncomingCapturedVideoFrame(
     std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer,
     scoped_refptr<VideoFrame> frame) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   const int buffer_id = buffer->id();
   DCHECK_NE(buffer_id, media::VideoCaptureBufferPool::kInvalidId);
 
-  int count = 0;
+  // Insert if not exists.
+  const auto it =
+      buffer_id_to_state_map_
+          .insert(std::make_pair(
+              buffer_id, BufferState(buffer_id, buffer->frame_feedback_id(),
+                                     consumer_feedback_observer_.get(),
+                                     buffer_pool_, frame)))
+          .first;
+  BufferState& buffer_state = it->second;
+  DCHECK(buffer_state.HasZeroConsumerHoldCount());
+
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
       frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
                                    video_capture_format_.frame_rate);
     }
     std::unique_ptr<base::DictionaryValue> metadata(
         new base::DictionaryValue());
     frame->metadata()->MergeInternalValuesInto(metadata.get());
 
     // Only I420 and Y16 pixel formats are currently supported.
     DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
            frame->format() == media::PIXEL_FORMAT_Y16)
         << "Unsupported pixel format: "
         << media::VideoPixelFormatToString(frame->format());
 
     // Sanity-checks to confirm |frame| is actually being backed by |buffer|.
     DCHECK(frame->storage_type() == media::VideoFrame::STORAGE_SHMEM);
     DCHECK(frame->data(media::VideoFrame::kYPlane) >= buffer->data(0) &&
            (frame->data(media::VideoFrame::kYPlane) <
             (reinterpret_cast<const uint8_t*>(buffer->data(0)) +
              buffer->mapped_size())))
         << "VideoFrame does not appear to be backed by Buffer";
 
     for (const auto& client : controller_clients_) {
       if (client->session_closed || client->paused)
         continue;
 
       // On the first use of a buffer on a client, share the memory handles.
-      const bool is_new_buffer = client->known_buffers.insert(buffer_id).second;
+      auto known_buffers_entry_iter =
+          std::find(std::begin(client->known_buffers),
+                    std::end(client->known_buffers), buffer_id);
+      bool is_new_buffer = false;
+      if (known_buffers_entry_iter == std::end(client->known_buffers)) {
+        client->known_buffers.push_back(buffer_id);
+        is_new_buffer = true;
+      }
       if (is_new_buffer)
         DoNewBufferOnIOThread(client.get(), buffer.get(), frame);
 
       client->event_handler->OnBufferReady(client->controller_id, buffer_id,
                                            frame);
-      const bool inserted =
-          client->active_buffers.insert(std::make_pair(buffer_id, frame))
-              .second;
-      DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer_id;
-      count++;
+
+      auto buffers_in_use_entry_iter =
+          std::find(std::begin(client->buffers_in_use),
+                    std::end(client->buffers_in_use), buffer_id);
+      if (buffers_in_use_entry_iter == std::end(client->buffers_in_use))
+        client->buffers_in_use.push_back(buffer_id);
+      else
+        DCHECK(false) << "Unexpected duplicate buffer: " << buffer_id;
+      buffer_state.IncreaseConsumerCount();
     }
   }
 
   if (!has_received_frames_) {
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
                          frame->visible_rect().width());
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
                          frame->visible_rect().height());
     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
                                frame->visible_rect().width(),
                                frame->visible_rect().height());
     double frame_rate = 0.0f;
     if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
                                       &frame_rate)) {
       frame_rate = video_capture_format_.frame_rate;
     }
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
     has_received_frames_ = true;
   }
-
-  buffer_pool_->HoldForConsumers(buffer_id, count);
 }
 
 void VideoCaptureController::OnError() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
        continue;
     client->event_handler->OnError(client->controller_id);
   }
 }
 
 void VideoCaptureController::OnLog(const std::string& message) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 }
 
 void VideoCaptureController::OnBufferDestroyed(int buffer_id_to_drop) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
       continue;
 
-    if (client->known_buffers.erase(buffer_id_to_drop)) {
+    auto known_buffers_entry_iter =
+        std::find(std::begin(client->known_buffers),
+                  std::end(client->known_buffers), buffer_id_to_drop);
+    if (known_buffers_entry_iter != std::end(client->known_buffers)) {
+      client->known_buffers.erase(known_buffers_entry_iter);
       client->event_handler->OnBufferDestroyed(client->controller_id,
                                                buffer_id_to_drop);
     }
   }
+
+  buffer_id_to_state_map_.erase(buffer_id_to_drop);
 }
 
 void VideoCaptureController::DoNewBufferOnIOThread(
     ControllerClient* client,
     media::VideoCaptureDevice::Client::Buffer* buffer,
     const scoped_refptr<media::VideoFrame>& frame) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK_EQ(media::VideoFrame::STORAGE_SHMEM, frame->storage_type());
 
   const int buffer_id = buffer->id();
@@ skipping 19 matching lines @@
     int session_id,
     const ControllerClients& clients) {
   for (const auto& client : clients) {
     if (client->session_id == session_id)
       return client.get();
   }
   return nullptr;
 }
 
 }  // namespace content