[Mojo Video Capture] Replace RESOURCE_UTILIZATION with interface ReceiverLoadObserver

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <map>
@@ skipping 28 matching lines @@
 
 namespace {
 
 static const int kInfiniteRatio = 99999;
 
 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
     UMA_HISTOGRAM_SPARSE_SLOWLY( \
         name, \
         (height) ? ((width) * 100) / (height) : kInfiniteRatio);
 
-class SyncTokenClientImpl : public VideoFrame::SyncTokenClient {
- public:
-  explicit SyncTokenClientImpl(display_compositor::GLHelper* gl_helper)
-      : gl_helper_(gl_helper) {}
-  ~SyncTokenClientImpl() override {}
-  void GenerateSyncToken(gpu::SyncToken* sync_token) override {
-    gl_helper_->GenerateSyncToken(sync_token);
-  }
-  void WaitSyncToken(const gpu::SyncToken& sync_token) override {
-    gl_helper_->WaitSyncToken(sync_token);
-  }
-
- private:
-  display_compositor::GLHelper* gl_helper_;
-};
-
-void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
-                      const gpu::SyncToken& sync_token) {
-  DCHECK_CURRENTLY_ON(BrowserThread::UI);
-#if defined(OS_ANDROID)
-  NOTREACHED();
-#else
-  display_compositor::GLHelper* gl_helper =
-      ImageTransportFactory::GetInstance()->GetGLHelper();
-  // UpdateReleaseSyncToken() creates a new sync_token using |gl_helper|, so
-  // wait the given |sync_token| using |gl_helper|.
-  if (gl_helper) {
-    gl_helper->WaitSyncToken(sync_token);
-    SyncTokenClientImpl client(gl_helper);
-    video_frame->UpdateReleaseSyncToken(&client);
-  }
-#endif
-}
-
 std::unique_ptr<media::VideoCaptureJpegDecoder> CreateGpuJpegDecoder(
     const media::VideoCaptureJpegDecoder::DecodeDoneCB& decode_done_cb) {
   return base::MakeUnique<VideoCaptureGpuJpegDecoder>(decode_done_cb);
 }
 
 // Decorator for media::VideoFrameReceiver that forwards all incoming calls
 // to the Browser IO thread.
 class VideoFrameReceiverOnIOThread : public media::VideoFrameReceiver {
  public:
   explicit VideoFrameReceiverOnIOThread(
@@ skipping 50 matching lines @@
   // ID used for identifying this object.
   const VideoCaptureControllerID controller_id;
   VideoCaptureControllerEventHandler* const event_handler;
 
   const media::VideoCaptureSessionId session_id;
   const media::VideoCaptureParams parameters;
 
   // Buffers that are currently known to this client.
   std::set<int> known_buffers;
 
-  // Buffers currently held by this client, and sync token callback to call when
-  // they are returned from the client.
-  typedef std::map<int, scoped_refptr<VideoFrame>> ActiveBufferMap;
-  ActiveBufferMap active_buffers;
+  // Buffers currently held by this client.
+  std::set<int> buffers_in_use;
 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
   // client is sent an OnEnded() event. However, because the client retains a
   // VideoCaptureController* pointer, its ControllerClient entry lives on until
   // it unregisters itself via RemoveClient(), which may happen asynchronously.
   //
   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
   // OnEnded() events were processed synchronously (with the RemoveClient() done
   // implicitly), we could avoid tracking this state here in the Controller, and
   // simplify the code in both places.
   bool session_closed;
 
   // Indicates whether the client is paused, if true, VideoCaptureController
   // stops updating its buffer.
   bool paused;
 };
 
+VideoCaptureController::BufferState::BufferState()
+    : buffer_id_(0),
+      frame_receiver_observer_(nullptr),
+      max_consumer_utilization_(0),
+      consumer_hold_count_(0) {}
+
+VideoCaptureController::BufferState::BufferState(
+    int buffer_id,
+    media::FrameReceiverObserver* frame_receiver_observer,
+    const scoped_refptr<media::VideoCaptureBufferPool>& buffer_pool,
+    const scoped_refptr<media::VideoFrame>& frame)
+    : buffer_id_(buffer_id),
+      frame_receiver_observer_(frame_receiver_observer),
+      buffer_pool_(buffer_pool),
+      frame_(frame),
+      max_consumer_utilization_(
+          media::FrameReceiverObserver::no_utilization_recorded()),
+      consumer_hold_count_(0) {}
+
+VideoCaptureController::BufferState::BufferState(
+    VideoCaptureController::BufferState&& other) = default;
+
+VideoCaptureController::BufferState::~BufferState() = default;
+
+VideoCaptureController::BufferState& VideoCaptureController::BufferState::
+operator=(VideoCaptureController::BufferState&& other) = default;
+
+void VideoCaptureController::BufferState::RecordConsumerUtilization(
+    double utilization) {
+  if (std::isfinite(utilization) && utilization >= 0.0) {
+    max_consumer_utilization_ =
+        std::max(max_consumer_utilization_, utilization);
+  }
+}
+
+void VideoCaptureController::BufferState::IncreaseConsumerCount() {
+  if (consumer_hold_count_ == 0)
+    buffer_pool_->HoldForConsumers(buffer_id_, 1);

[miu, 2016/12/01 05:25:17]

Looks like the "hold count" tracking isn't needed in the buffer pool anymore.
Future cleanup?

[chfremer, 2016/12/02 01:28:28]

Excellent observation. I put it on the list of future cleanups in the design
doc.
Actually, with one of the follow-up CLs we may even no longer need to
differentiate between consumer hold and producer hold (but I am getting ahead of
myself here). We may be able to do apply both of those simplifications to the
buffer pool in one go.

+  consumer_hold_count_++;
+}
+
+void VideoCaptureController::BufferState::DecreaseConsumerCount() {
+  consumer_hold_count_--;
+  if (consumer_hold_count_ == 0) {
+    if (frame_receiver_observer_ != nullptr &&
+        max_consumer_utilization_ !=
+            media::FrameReceiverObserver::no_utilization_recorded()) {
+      frame_receiver_observer_->OnReceiverReportingUtilization(
+          buffer_id_, max_consumer_utilization_);
+    }
+    buffer_pool_->RelinquishConsumerHold(buffer_id_, 1);
+    max_consumer_utilization_ =
+        media::FrameReceiverObserver::no_utilization_recorded();
+  }
+}
+
+bool VideoCaptureController::BufferState::HasZeroConsumerHoldCount() {
+  return consumer_hold_count_ == 0;
+}
+
 VideoCaptureController::VideoCaptureController(int max_buffers)
     : buffer_pool_(new media::VideoCaptureBufferPoolImpl(
           base::MakeUnique<media::VideoCaptureBufferTrackerFactoryImpl>(),
           max_buffers)),
+      frame_receiver_observer_(nullptr),
       state_(VIDEO_CAPTURE_STATE_STARTED),
       has_received_frames_(false),
       weak_ptr_factory_(this) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 }
 
 base::WeakPtr<VideoCaptureController>
 VideoCaptureController::GetWeakPtrForIOThread() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
-std::unique_ptr<media::VideoCaptureDevice::Client>
+void VideoCaptureController::SetFrameReceiverObserver(
+    media::FrameReceiverObserver* frame_receiver_observer) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  frame_receiver_observer_ = frame_receiver_observer;
+}
+
+std::unique_ptr<media::VideoCaptureDeviceClient>
 VideoCaptureController::NewDeviceClient() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return base::MakeUnique<media::VideoCaptureDeviceClient>(
       base::MakeUnique<VideoFrameReceiverOnIOThread>(
           this->GetWeakPtrForIOThread()),
       buffer_pool_,
       base::Bind(&CreateGpuJpegDecoder,
                  base::Bind(&VideoFrameReceiver::OnIncomingCapturedVideoFrame,
-                            this->GetWeakPtrForIOThread())));
+                            this->GetWeakPtrForIOThread())),
+      base::ThreadTaskRunnerHandle::Get());
 }
 
 void VideoCaptureController::AddClient(
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* event_handler,
     media::VideoCaptureSessionId session_id,
     const media::VideoCaptureParams& params) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DVLOG(1) << "VideoCaptureController::AddClient() -- id=" << id
            << ", session_id=" << session_id
@@ skipping 42 matching lines @@
     VideoCaptureControllerID id,
     VideoCaptureControllerEventHandler* event_handler) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id;
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
   if (!client)
     return kInvalidMediaCaptureSessionId;
 
   // Take back all buffers held by the |client|.
-  for (const auto& buffer : client->active_buffers)
-    buffer_pool_->RelinquishConsumerHold(buffer.first, 1);
-  client->active_buffers.clear();
+  for (const auto& buffer_id : client->buffers_in_use)
+    buffer_id_to_state_map_[buffer_id].DecreaseConsumerCount();
+  client->buffers_in_use.clear();
 
   int session_id = client->session_id;
   controller_clients_.remove_if(
       [client](const std::unique_ptr<ControllerClient>& ptr) {
         return ptr.get() == client;
       });
 
   return session_id;
 }
 
@@ skipping 71 matching lines @@
     VideoCaptureControllerEventHandler* event_handler,
     int buffer_id,
     const gpu::SyncToken& sync_token,
     double consumer_resource_utilization) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
-  ControllerClient::ActiveBufferMap::iterator iter;
-  if (!client || (iter = client->active_buffers.find(buffer_id)) ==
-                     client->active_buffers.end()) {
+  if (!client || (client->buffers_in_use.find(buffer_id) ==
+                  client->buffers_in_use.end())) {
     NOTREACHED();
     return;
   }
 
-  // Set the RESOURCE_UTILIZATION to the maximum of those provided by each
-  // consumer (via separate calls to this method that refer to the same
-  // VideoFrame).  The producer of this VideoFrame may check this value, after
-  // all consumer holds are relinquished, to make quality versus performance
-  // trade-off decisions.
-  scoped_refptr<VideoFrame> frame = iter->second;
-  if (std::isfinite(consumer_resource_utilization) &&
-      consumer_resource_utilization >= 0.0) {
-    double resource_utilization = -1.0;
-    if (frame->metadata()->GetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                     &resource_utilization)) {
-      frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                   std::max(consumer_resource_utilization,
-                                            resource_utilization));
-    } else {
-      frame->metadata()->SetDouble(VideoFrameMetadata::RESOURCE_UTILIZATION,
-                                   consumer_resource_utilization);
-    }
-  }
-
-  client->active_buffers.erase(iter);
-  buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
-
-#if defined(OS_ANDROID)
-  DCHECK(!sync_token.HasData());
-#endif
-  if (sync_token.HasData())
-    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
-                            base::Bind(&ReturnVideoFrame, frame, sync_token));
+  BufferState& buffer_state = buffer_id_to_state_map_[buffer_id];
+  buffer_state.RecordConsumerUtilization(consumer_resource_utilization);
+  buffer_state.DecreaseConsumerCount();
+  client->buffers_in_use.erase(buffer_id);
 }
 
 const media::VideoCaptureFormat&
 VideoCaptureController::GetVideoCaptureFormat() const {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return video_capture_format_;
 }
 
 VideoCaptureController::~VideoCaptureController() {
 }
 
 void VideoCaptureController::OnIncomingCapturedVideoFrame(
     std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> buffer,
     scoped_refptr<VideoFrame> frame) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   const int buffer_id = buffer->id();
   DCHECK_NE(buffer_id, media::VideoCaptureBufferPool::kInvalidId);
 
-  int count = 0;
+  if (buffer_id_to_state_map_.find(buffer_id) ==

[miu, 2016/12/01 05:25:17]

As mentioned in the header file, you might be able to eliminate the zero-arg
ctor and 3 map look-ups here by using iterators:

  // Insert if not exists.
  const auto it = buffer_id_to_state_map_.insert(
      std::make_pair(buffer_id, BufferState(...))).first;
  BufferState& buffer_state = it->second;

[chfremer, 2016/12/02 01:28:28]

Very nice solution. Thanks.
Unfortunately, in order to make the invariant members of BufferState const, we
now have to make a copy, but I think it is worth it. If only our compiler would
support std::map::emplace...

+         buffer_id_to_state_map_.end()) {
+    BufferState new_buffer_state(buffer_id, frame_receiver_observer_,
+                                 buffer_pool_, frame);
+    buffer_id_to_state_map_[buffer_id] = std::move(new_buffer_state);
+  }
+  BufferState& buffer_state = buffer_id_to_state_map_[buffer_id];
+  DCHECK(buffer_state.HasZeroConsumerHoldCount());
+
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     if (!frame->metadata()->HasKey(VideoFrameMetadata::FRAME_RATE)) {
       frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE,
                                    video_capture_format_.frame_rate);
     }
     std::unique_ptr<base::DictionaryValue> metadata(
         new base::DictionaryValue());
     frame->metadata()->MergeInternalValuesInto(metadata.get());
 
     // Only I420 and Y16 pixel formats are currently supported.
@@ skipping 14 matching lines @@
       if (client->session_closed || client->paused)
         continue;
 
       // On the first use of a buffer on a client, share the memory handles.
       const bool is_new_buffer = client->known_buffers.insert(buffer_id).second;
       if (is_new_buffer)
         DoNewBufferOnIOThread(client.get(), buffer.get(), frame);
 
       client->event_handler->OnBufferReady(client->controller_id, buffer_id,
                                            frame);
-      const bool inserted =
-          client->active_buffers.insert(std::make_pair(buffer_id, frame))
-              .second;
+      const bool inserted = client->buffers_in_use.insert(buffer_id).second;
       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer_id;
-      count++;
+      buffer_state.IncreaseConsumerCount();
     }
   }
 
   if (!has_received_frames_) {
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Width",
                          frame->visible_rect().width());
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.Height",
                          frame->visible_rect().height());
     UMA_HISTOGRAM_ASPECT_RATIO("Media.VideoCapture.AspectRatio",
                                frame->visible_rect().width(),
                                frame->visible_rect().height());
     double frame_rate = 0.0f;
     if (!frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE,
                                       &frame_rate)) {
       frame_rate = video_capture_format_.frame_rate;
     }
     UMA_HISTOGRAM_COUNTS("Media.VideoCapture.FrameRate", frame_rate);
     has_received_frames_ = true;
   }
-
-  buffer_pool_->HoldForConsumers(buffer_id, count);
 }
 
 void VideoCaptureController::OnError() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
        continue;
     client->event_handler->OnError(client->controller_id);
@@ skipping 10 matching lines @@
 
   for (const auto& client : controller_clients_) {
     if (client->session_closed)
       continue;
 
     if (client->known_buffers.erase(buffer_id_to_drop)) {
       client->event_handler->OnBufferDestroyed(client->controller_id,
                                                buffer_id_to_drop);
     }
   }
+
+  buffer_id_to_state_map_.erase(buffer_id_to_drop);
 }
 
 void VideoCaptureController::DoNewBufferOnIOThread(
     ControllerClient* client,
     media::VideoCaptureDevice::Client::Buffer* buffer,
     const scoped_refptr<media::VideoFrame>& frame) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK_EQ(media::VideoFrame::STORAGE_SHMEM, frame->storage_type());
 
   const int buffer_id = buffer->id();
@@ skipping 19 matching lines @@
     int session_id,
     const ControllerClients& clients) {
   for (const auto& client : clients) {
     if (client->session_id == session_id)
       return client.get();
   }
   return nullptr;
 }
 
 }  // namespace content