Adds volume level measurements to the AudioInputController for low-latency clients

media/audio/audio_input_controller.cc

Index: media/audio/audio_input_controller.cc
diff --git a/media/audio/audio_input_controller.cc b/media/audio/audio_input_controller.cc
index 3bdec149bfc3c3605bdb7234edf522199fedb783..afd2700da2cad521194b8ad1e3c7f94a6331e867 100644
--- a/media/audio/audio_input_controller.cc
+++ b/media/audio/audio_input_controller.cc
@@ -6,10 +6,13 @@
 
 #include "base/bind.h"
 #include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
 #include "media/base/limits.h"
 #include "media/base/scoped_histogram_timer.h"
 #include "media/base/user_input_monitor.h"
 
+using base::TimeDelta;
+
 namespace {
 const int kMaxInputChannels = 3;
 
@@ -25,6 +28,20 @@ const int kTimerResetIntervalSeconds = 1;
 // Mac devices and the initial timer interval has therefore been increased
 // from 1 second to 5 seconds.
 const int kTimerInitialIntervalSeconds = 5;
+
+// Time constant for AudioPowerMonitor.
+// The utilized smoothing factor (alpha) in the exponential filter is given
+// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
+// constant given by |kPowerMeasurementTimeConstantMilliseconds|.
+// Example: fs=44100, ts=10e-3 => alpha~0.022420
+//          fs=44100, ts=20e-3 => alpha~0.165903
+// A large smoothing factor corresponds to a faster filter response to input
+// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
+// and y(n) is the output.
+const int kPowerMeasurementTimeConstantMilliseconds = 10;
+
+// Time between two successive measurements of audio power levels.
+const int kPowerMonitorLogIntervalMilliseconds = 1000;
 }
 
 namespace media {
@@ -32,6 +49,24 @@ namespace media {
 // static
 AudioInputController::Factory* AudioInputController::factory_ = NULL;
 
+#if defined(AUDIO_POWER_MONITORING)
+// Calculate and log the audible level in the range [0.0,1.0], where 0.0
+// means the audio signal is silent and 1.0 means it is at maximum volume.
+// TODO(henrika): we should call MediaStreamManager::SendMessageToNativeLog()
+// here as well.
+static void AddPowerLevelToLog(float level_dbfs) {

[no longer working on chromium, 2014/05/23 08:29:56]

shouldn't you hook it up to the native log?

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Yes, just wanted OK about general stuff first. Will add that next round.

[henrika (OOO until Aug 14), 2014/05/26 11:02:26]

I removed this part and log in dBFS instead. Easier to compare with log in WAC
(in render process) then.

+    float level = 0;
+    static const float kSilenceThresholdDBFS = -72.24719896f;
+    if (level_dbfs < kSilenceThresholdDBFS)
+      level = 0.0f;
+    else if (level_dbfs > 0.0f)
+      level = 1.0f;
+    else
+      level = 1.0f - level_dbfs / kSilenceThresholdDBFS;
+    DVLOG(1) << "audio_level: " << level;
+}
+#endif
+
 AudioInputController::AudioInputController(EventHandler* handler,
                                            SyncWriter* sync_writer,
                                            UserInputMonitor* user_input_monitor)
@@ -59,6 +94,7 @@ scoped_refptr<AudioInputController> AudioInputController::Create(
     const std::string& device_id,
     UserInputMonitor* user_input_monitor) {
   DCHECK(audio_manager);
+  DVLOG(1) << "AudioInputController::Create";
 
   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     return NULL;
@@ -93,6 +129,7 @@ scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
     UserInputMonitor* user_input_monitor) {
   DCHECK(audio_manager);
   DCHECK(sync_writer);
+  DVLOG(1) << "AudioInputController::CreateLowLatency";
 
   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     return NULL;
@@ -173,6 +210,19 @@ void AudioInputController::DoCreate(AudioManager* audio_manager,
                                     const std::string& device_id) {
   DCHECK(task_runner_->BelongsToCurrentThread());
   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
+  DVLOG(1) << "AudioInputController::DoCreate";
+
+#if defined(AUDIO_POWER_MONITORING)
+  // Create the audio (power) level meter given the provided audio parameters.
+  // An AudioBus is also needed to wrap the raw data buffer from the native
+  // layer to match AudioPowerMonitor::Scan().
+  // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
+  audio_level_.reset(new media::AudioPowerMonitor(params.sample_rate(),
+      TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
+  audio_bus_ = AudioBus::Create(params);
+  audio_params_ = params;
+#endif
+
   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
   // platform audio input requires the |no_data_timer_| be used to auto-detect
   // errors.  In reality, probably only Windows needs to be treated as
@@ -266,6 +316,11 @@ void AudioInputController::DoClose() {
   // Delete the timer on the same thread that created it.
   no_data_timer_.reset();
 
+#if defined(AUDIO_POWER_MONITORING)
+  if (audio_level_)
+    audio_level_->Reset();

[no longer working on chromium, 2014/05/23 08:29:56]

I realized I was confused with Reset() and reset().

I think we should put a webrtc log here so that we know the stream is closed.

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Good point. I can do that.

[henrika (OOO until Aug 14), 2014/05/26 11:02:26]

Done.

+#endif
+
   DoStopCloseAndClearStream();
   SetDataIsActive(false);
 
@@ -377,6 +432,35 @@ void AudioInputController::OnData(AudioInputStream* stream,
   if (SharedMemoryAndSyncSocketMode()) {
     sync_writer_->Write(data, size, volume, key_pressed);
     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
+
+#if defined(AUDIO_POWER_MONITORING)
+    {
+      // Only do power-level measurements if an AudioPowerMonitor object has
+      // been created. Done in DoCreate() but not DoCreateForStream(), hence
+      // logging will mainly be done for WebRTC and WebSpeech clients.
+      base::AutoLock auto_lock(lock_);
+      if (!audio_level_)

[no longer working on chromium, 2014/05/23 08:29:56]

why this needs to be under the lock?

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

The callback is called on a native audio thread and not the AM thread on which
the AudioPowerMonitor was created. OK?

[no longer working on chromium, 2014/05/26 14:20:16]

not sure I follow here. In DoCreate(), the creation of audio_level_ is not under
the lock, so this lock protects nothing here.

Also, audio_level_ is created before the stream is create, and OnData() is
triggered when the stream is active. And audio_level_ is deleted before AIC goes
away. theoretically audio_level_ should be always valid here.

Or do I miss something?

+        return;
+    }
+
+    // Perform periodic audio (power) level measurements.
+    if ((base::TimeTicks::Now() - last_audio_level_log_time_).InMilliseconds() >
+        kPowerMonitorLogIntervalMilliseconds) {

[no longer working on chromium, 2014/05/23 08:29:56]

indentation

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Done.

[henrika (OOO until Aug 14), 2014/05/26 11:02:26]

Actually; this is what I get when using git cl format.

+      // Wrap data into and AudioBus to match AudioPowerMonitor::Scan.
+      audio_bus_->FromInterleaved(

[no longer working on chromium, 2014/05/23 08:29:56]

please add a todo here to remind switching to AudioBus for input and remove the
FromInterleaved here.

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Will do.

+          data, audio_bus_->frames(), audio_params_.bits_per_sample() / 8);
+      audio_level_->Scan(*audio_bus_, audio_bus_->frames());
+
+      // Get current average power level and add it to the log.
+      std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
+      AddPowerLevelToLog(result.first);
+
+      // Reset the average power level (since we don't log continuously).
+      audio_level_->Reset();

[no longer working on chromium, 2014/05/23 08:29:56]

I am not familiar with AudioPowerMonitor, do you know how this Reset affect the
smoothing factor ? for example, if the frame size in audio_bus is 3ms,
kPowerMeasurementTimeConstantMilliseconds is 10, but you Reset the audio level
after scanning one frame, does this still work?

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Yes it works. The smoothing factor controls how fast the filter will detect that
the signal changes but given that we only filter once every other second it
makes no sense to keep the old value in memory. 

Regarding 3ms, let me run some more tests on Mac and check. Let me think some
more on this.

+      last_audio_level_log_time_ = base::TimeTicks::Now();
+    }
+#endif
+
     return;
   }