[AudioStreamMonitor] Adds API to collect frame-level audibility.

content/browser/media/audio_stream_monitor.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/media/audio_stream_monitor.h"
 
+#include <unordered_map>
+
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "content/browser/web_contents/web_contents_impl.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/invalidate_type.h"
 #include "content/public/browser/render_frame_host.h"
 
 namespace content {
 
 namespace {
 
 enum class ActionType { STARTING, STOPPING };
 AudioStreamMonitor* StartStopMonitoringHelper(ActionType action_type,
                                               int render_process_id,
                                               int render_frame_id) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   // It's important that this code uses only the process id for lookup as there
   // may not be a RenderFrameHost or WebContents attached to the RenderProcess
   // at time of call; e.g., in the event of a crash.
   RenderProcessHost* const render_process_host =
       RenderProcessHost::FromID(render_process_id);
   if (!render_process_host)
     return nullptr;
 
-  // TODO(dalecurtis, maxmorin): We should really only be sending these when the
-  // streams are audible or we don't have power level monitoring.
-  if (action_type == ActionType::STARTING)
-    render_process_host->OnAudioStreamAdded();
-  else
-    render_process_host->OnAudioStreamRemoved();
-
   WebContentsImpl* const web_contents =
       static_cast<WebContentsImpl*>(WebContents::FromRenderFrameHost(
           RenderFrameHost::FromID(render_process_id, render_frame_id)));
   return web_contents ? web_contents->audio_stream_monitor() : nullptr;
 }
 
 }  // namespace
 
 AudioStreamMonitor::AudioStreamMonitor(WebContents* contents)
     : web_contents_(contents),
@@ skipping 18 matching lines @@
 void AudioStreamMonitor::RenderProcessGone(int render_process_id) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   // Note: It's possible for the RenderProcessHost and WebContents (and thus
   // this class) to survive the death of the render process and subsequently be
   // reused. During this period StartStopMonitoringHelper() will be unable to
   // lookup the WebContents using the now-dead |render_frame_id|. We must thus
   // have this secondary mechanism for clearing stale callbacks.
 
   for (auto it = poll_callbacks_.begin(); it != poll_callbacks_.end();) {
-    if (it->first.first == render_process_id) {
+    if (std::get<0>(it->first) == render_process_id) {
       it = poll_callbacks_.erase(it);
       OnStreamRemoved();
     } else {
       ++it;
     }
   }
 
   if (poll_callbacks_.empty())
     poll_timer_.Stop();
 }
@@ skipping 27 matching lines @@
 
 // static
 void AudioStreamMonitor::StartMonitoringHelper(
     int render_process_id,
     int render_frame_id,
     int stream_id,
     const ReadPowerAndClipCallback& read_power_callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   if (AudioStreamMonitor* monitor = StartStopMonitoringHelper(
           ActionType::STARTING, render_process_id, render_frame_id)) {
-    monitor->StartMonitoringStreamOnUIThread(render_process_id, stream_id,
-                                             read_power_callback);
+    monitor->StartMonitoringStreamOnUIThread(render_process_id, render_frame_id,
+                                             stream_id, read_power_callback);
   }
 }
 
 // static
 void AudioStreamMonitor::StopMonitoringHelper(int render_process_id,
                                               int render_frame_id,
                                               int stream_id) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   if (AudioStreamMonitor* monitor = StartStopMonitoringHelper(
           ActionType::STOPPING, render_process_id, render_frame_id)) {
-    monitor->StopMonitoringStreamOnUIThread(render_process_id, stream_id);
+    monitor->StopMonitoringStreamOnUIThread(render_process_id, render_frame_id,
+                                            stream_id);
   }
 }
 
 void AudioStreamMonitor::StartMonitoringStreamOnUIThread(
     int render_process_id,
+    int render_frame_id,
     int stream_id,
     const ReadPowerAndClipCallback& read_power_callback) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(!read_power_callback.is_null());
 
-  const StreamID qualified_id(render_process_id, stream_id);
+  const StreamID qualified_id(render_process_id, render_frame_id, stream_id);
   DCHECK(poll_callbacks_.find(qualified_id) == poll_callbacks_.end());
 
   poll_callbacks_[qualified_id] = read_power_callback;
+  // We should send audio-added signal to RenderProcessHost when we don't have

[miu, 2017/06/19 20:44:01]

style nit: Try not to use "we" in comments. :)

[lpy, 2017/06/20 00:19:21]

Done.

+  // power level monitoring, otherwise we should send the signal when the stream
+  // becomes non-audible.
+  if (!power_level_monitoring_available())
+    RenderProcessHost::FromID(render_process_id)->OnAudioStreamAdded();

[miu, 2017/06/19 20:44:01]

Always null-check the result of RPH::FromID() (here and elsewhere), since you
never know when a process will go away. Think of the |render_frame_id| as a weak
pointer. I usually write code like this:

  if (auto* host = RenderProcesHost::FromId(...))
    host->OnAudioStreamAdded();

[lpy, 2017/06/20 00:19:21]

Done.

   OnStreamAdded();
 }
 
 void AudioStreamMonitor::StopMonitoringStreamOnUIThread(int render_process_id,
+                                                        int render_frame_id,
                                                         int stream_id) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   // In the event of render process death, these may have already been cleared.
-  auto it = poll_callbacks_.find(StreamID(render_process_id, stream_id));
+  auto it = poll_callbacks_.find(
+      StreamID(render_process_id, render_frame_id, stream_id));
   if (it == poll_callbacks_.end())
     return;
 
   poll_callbacks_.erase(it);
+  // We should send audio-removed signal to RenderProcessHost when we don't have

[miu, 2017/06/19 20:44:01]

style nit: ditto (no "we")

[lpy, 2017/06/20 00:19:21]

Done.

+  // power level monitoring, otherwise we should send the signal when the stream
+  // becomes non-audible.
+  if (!power_level_monitoring_available())
+    RenderProcessHost::FromID(render_process_id)->OnAudioStreamRemoved();
   OnStreamRemoved();
 }
 
 void AudioStreamMonitor::Poll() {
   bool was_audible = is_audible_;
   is_audible_ = false;
 
+  // Record whether or not a RenderFrameHost is audible.
+  std::unordered_map<RenderFrameHost*, bool> is_frame_audible;

[miu, 2017/06/19 20:44:01]

It feels expensive to build-up a data structure on each Poll() call (several per
second). I'm trying to think of whether there might be lighter-weight approaches
to accomplish our goals here.

Can you point us to a design doc? We should consider how the various audio
playback cases will determine when/how each render frame is throttled. For
example, if we want a "throttle-down after N seconds of silence" heuristic, then
we may only need to record the |last_blurt_time_| per render frame.

[lpy, 2017/06/20 00:19:21]

https://docs.google.com/document/d/15wWebVCLU3UM-OAQBVi1mOX0ZPzYupPN9tw4wy7lb...

As an example, a metric we want to collect, is the duration from a tab becomes
backgrounded to a frame(say root frame) in it becomes audible, we consider a
frame becomes audible if it's not audible in the last 1 minute.

I think that's a good suggestion. What about sending audio-added/removed signal
to RenderProcessHost? It's really tricky in RenderProcessGone(), because we
don't want to call RenderProcessHost::OnAudioStreamRemoved() everytime, we only
want to call it when RenderFrameHost is audible, and since the |poll_callbacks_|
can have multiple entries for one render frame host, we also want to avoid
calling multiple times on RenderProcessHost, we want to call just one time for a
render frame host. Do I understand correctly?

   for (StreamPollCallbackMap::const_iterator it = poll_callbacks_.begin();
-       it != poll_callbacks_.end();
-       ++it) {
+       it != poll_callbacks_.end(); ++it) {
     // TODO(miu): A new UI for delivering specific power level and clipping
     // information is still in the works.  For now, we throw away all
     // information except for "is it audible?"
     const float power_dbfs = it->second.Run().first;
     const float kSilenceThresholdDBFS = -72.24719896f;
 
-    if (power_dbfs >= kSilenceThresholdDBFS) {
+    bool is_stream_audible = power_dbfs >= kSilenceThresholdDBFS;
+    if (!is_audible_ && is_stream_audible) {
       last_blurt_time_ = clock_->NowTicks();
       is_audible_ = true;
       MaybeToggle();
-      break;  // No need to poll remaining streams.
+    }
+
+    // Record whether or not the RenderFrame is audible, a RenderFrame is
+    // audible when there is a audio stream in it that is audible.
+    auto* render_frame_host =
+        RenderFrameHost::FromID(std::get<0>(it->first), std::get<1>(it->first));
+    if (!render_frame_host)
+      continue;
+    if (is_frame_audible.find(render_frame_host) == is_frame_audible.end()) {
+      is_frame_audible[render_frame_host] = is_stream_audible;
+    } else {
+      is_frame_audible[render_frame_host] |= is_stream_audible;
     }
   }
 
+  // Update RenderFrameHost and RenderProcessHost audible state only when state
+  // changed.
+  for (auto it = is_frame_audible.begin(); it != is_frame_audible.end(); ++it) {
+    auto* render_frame_host = it->first;
+    bool is_frame_audible = it->second;
+    if (is_frame_audible != render_frame_host->IsAudible()) {
+      render_frame_host->OnAudioStateChanged(is_frame_audible);
+      if (is_frame_audible) {
+        render_frame_host->GetProcess()->OnAudioStreamAdded();
+      } else {
+        render_frame_host->GetProcess()->OnAudioStreamRemoved();
+      }
+    }
+  }
+
   if (is_audible_ != was_audible)
     web_contents_->OnAudioStateChanged(is_audible_);
 }
 
 void AudioStreamMonitor::MaybeToggle() {
   const bool indicator_was_on = was_recently_audible_;
   const base::TimeTicks off_time =
       last_blurt_time_ + base::TimeDelta::FromMilliseconds(kHoldOnMilliseconds);
   const base::TimeTicks now = clock_->NowTicks();
   const bool should_indicator_be_on = now < off_time;
@@ skipping 38 matching lines @@
   if (!power_level_monitoring_available()) {
     is_audible_ = false;
     web_contents_->OnAudioStateChanged(false);
     MaybeToggle();
   } else {
     poll_timer_.Stop();
   }
 }
 
 }  // namespace content