Adds volume level measurements to the AudioInputController for low-latency clients

media/audio/audio_input_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/audio/audio_input_controller.h"
 
 #include "base/bind.h"
 #include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
 #include "media/base/limits.h"
 #include "media/base/scoped_histogram_timer.h"
 #include "media/base/user_input_monitor.h"
 
+using base::TimeDelta;
+
 namespace {
 const int kMaxInputChannels = 3;
 
 // TODO(henrika): remove usage of timers and add support for proper
 // notification of when the input device is removed.  This was originally added
 // to resolve http://crbug.com/79936 for Windows platforms.  This then caused
 // breakage (very hard to repro bugs!) on other platforms: See
 // http://crbug.com/226327 and http://crbug.com/230972.
 // See also that the timer has been disabled on Mac now due to
 // crbug.com/357501.
 const int kTimerResetIntervalSeconds = 1;
 // We have received reports that the timer can be too trigger happy on some
 // Mac devices and the initial timer interval has therefore been increased
 // from 1 second to 5 seconds.
 const int kTimerInitialIntervalSeconds = 5;
+
+// Time constant for AudioPowerMonitor.
+// The utilized smoothing factor (alpha) in the exponential filter is given
+// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
+// constant given by |kPowerMeasurementTimeConstantMilliseconds|.
+// Example: fs=44100, ts=10e-3 => alpha~0.022420
+//          fs=44100, ts=20e-3 => alpha~0.165903
+// A large smoothing factor corresponds to a faster filter response to input
+// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
+// and y(n) is the output.
+const int kPowerMeasurementTimeConstantMilliseconds = 10;
+
+// Time between two successive measurements of audio power levels.
+const int kPowerMonitorLogIntervalMilliseconds = 1000;
 }
 
 namespace media {
 
 // static
 AudioInputController::Factory* AudioInputController::factory_ = NULL;
 
+#if defined(AUDIO_POWER_MONITORING)
+// Calculate and log the audible level in the range [0.0,1.0], where 0.0
+// means the audio signal is silent and 1.0 means it is at maximum volume.
+// TODO(henrika): we should call MediaStreamManager::SendMessageToNativeLog()
+// here as well.
+static void AddPowerLevelToLog(float level_dbfs) {

[no longer working on chromium, 2014/05/23 08:29:56]

shouldn't you hook it up to the native log?

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Yes, just wanted OK about general stuff first. Will add that next round.

[henrika (OOO until Aug 14), 2014/05/26 11:02:26]

I removed this part and log in dBFS instead. Easier to compare with log in WAC
(in render process) then.

+    float level = 0;
+    static const float kSilenceThresholdDBFS = -72.24719896f;
+    if (level_dbfs < kSilenceThresholdDBFS)
+      level = 0.0f;
+    else if (level_dbfs > 0.0f)
+      level = 1.0f;
+    else
+      level = 1.0f - level_dbfs / kSilenceThresholdDBFS;
+    DVLOG(1) << "audio_level: " << level;
+}
+#endif
+
 AudioInputController::AudioInputController(EventHandler* handler,
                                            SyncWriter* sync_writer,
                                            UserInputMonitor* user_input_monitor)
     : creator_task_runner_(base::MessageLoopProxy::current()),
       handler_(handler),
       stream_(NULL),
       data_is_active_(false),
       state_(CLOSED),
       sync_writer_(sync_writer),
       max_volume_(0.0),
       user_input_monitor_(user_input_monitor),
       prev_key_down_count_(0) {
   DCHECK(creator_task_runner_.get());
 }
 
 AudioInputController::~AudioInputController() {
   DCHECK_EQ(state_, CLOSED);
 }
 
 // static
 scoped_refptr<AudioInputController> AudioInputController::Create(
     AudioManager* audio_manager,
     EventHandler* event_handler,
     const AudioParameters& params,
     const std::string& device_id,
     UserInputMonitor* user_input_monitor) {
   DCHECK(audio_manager);
+  DVLOG(1) << "AudioInputController::Create";
 
   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     return NULL;
 
   if (factory_) {
     return factory_->Create(
         audio_manager, event_handler, params, user_input_monitor);
   }
   scoped_refptr<AudioInputController> controller(
       new AudioInputController(event_handler, NULL, user_input_monitor));
@@ skipping 14 matching lines @@
 // static
 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
     AudioManager* audio_manager,
     EventHandler* event_handler,
     const AudioParameters& params,
     const std::string& device_id,
     SyncWriter* sync_writer,
     UserInputMonitor* user_input_monitor) {
   DCHECK(audio_manager);
   DCHECK(sync_writer);
+  DVLOG(1) << "AudioInputController::CreateLowLatency";
 
   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     return NULL;
 
   // Create the AudioInputController object and ensure that it runs on
   // the audio-manager thread.
   scoped_refptr<AudioInputController> controller(
       new AudioInputController(event_handler, sync_writer, user_input_monitor));
   controller->task_runner_ = audio_manager->GetTaskRunner();
 
@@ skipping 60 matching lines @@
 void AudioInputController::SetAutomaticGainControl(bool enabled) {
   task_runner_->PostTask(FROM_HERE, base::Bind(
       &AudioInputController::DoSetAutomaticGainControl, this, enabled));
 }
 
 void AudioInputController::DoCreate(AudioManager* audio_manager,
                                     const AudioParameters& params,
                                     const std::string& device_id) {
   DCHECK(task_runner_->BelongsToCurrentThread());
   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
+  DVLOG(1) << "AudioInputController::DoCreate";
+
+#if defined(AUDIO_POWER_MONITORING)
+  // Create the audio (power) level meter given the provided audio parameters.
+  // An AudioBus is also needed to wrap the raw data buffer from the native
+  // layer to match AudioPowerMonitor::Scan().
+  // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
+  audio_level_.reset(new media::AudioPowerMonitor(params.sample_rate(),
+      TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
+  audio_bus_ = AudioBus::Create(params);
+  audio_params_ = params;
+#endif
+
   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
   // platform audio input requires the |no_data_timer_| be used to auto-detect
   // errors.  In reality, probably only Windows needs to be treated as
   // unreliable here.
   DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id),
                     true);
 }
 
 void AudioInputController::DoCreateForStream(
     AudioInputStream* stream_to_control, bool enable_nodata_timer) {
@@ skipping 73 matching lines @@
 void AudioInputController::DoClose() {
   DCHECK(task_runner_->BelongsToCurrentThread());
   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
 
   if (state_ == CLOSED)
     return;
 
   // Delete the timer on the same thread that created it.
   no_data_timer_.reset();
 
+#if defined(AUDIO_POWER_MONITORING)
+  if (audio_level_)
+    audio_level_->Reset();

[no longer working on chromium, 2014/05/23 08:29:56]

I realized I was confused with Reset() and reset().

I think we should put a webrtc log here so that we know the stream is closed.

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Good point. I can do that.

[henrika (OOO until Aug 14), 2014/05/26 11:02:26]

Done.

+#endif
+
   DoStopCloseAndClearStream();
   SetDataIsActive(false);
 
   if (SharedMemoryAndSyncSocketMode())
     sync_writer_->Close();
 
   if (user_input_monitor_)
     user_input_monitor_->DisableKeyPressMonitoring();
 
   state_ = CLOSED;
@@ skipping 91 matching lines @@
     key_pressed = current_count != prev_key_down_count_;
     prev_key_down_count_ = current_count;
     DVLOG_IF(6, key_pressed) << "Detected keypress.";
   }
 
   // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
   // Used by all low-latency clients except WebSpeech.
   if (SharedMemoryAndSyncSocketMode()) {
     sync_writer_->Write(data, size, volume, key_pressed);
     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
+
+#if defined(AUDIO_POWER_MONITORING)
+    {
+      // Only do power-level measurements if an AudioPowerMonitor object has
+      // been created. Done in DoCreate() but not DoCreateForStream(), hence
+      // logging will mainly be done for WebRTC and WebSpeech clients.
+      base::AutoLock auto_lock(lock_);
+      if (!audio_level_)

[no longer working on chromium, 2014/05/23 08:29:56]

why this needs to be under the lock?

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

The callback is called on a native audio thread and not the AM thread on which
the AudioPowerMonitor was created. OK?

[no longer working on chromium, 2014/05/26 14:20:16]

not sure I follow here. In DoCreate(), the creation of audio_level_ is not under
the lock, so this lock protects nothing here.

Also, audio_level_ is created before the stream is create, and OnData() is
triggered when the stream is active. And audio_level_ is deleted before AIC goes
away. theoretically audio_level_ should be always valid here.

Or do I miss something?

+        return;
+    }
+
+    // Perform periodic audio (power) level measurements.
+    if ((base::TimeTicks::Now() - last_audio_level_log_time_).InMilliseconds() >
+        kPowerMonitorLogIntervalMilliseconds) {

[no longer working on chromium, 2014/05/23 08:29:56]

indentation

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Done.

[henrika (OOO until Aug 14), 2014/05/26 11:02:26]

Actually; this is what I get when using git cl format.

+      // Wrap data into and AudioBus to match AudioPowerMonitor::Scan.
+      audio_bus_->FromInterleaved(

[no longer working on chromium, 2014/05/23 08:29:56]

please add a todo here to remind switching to AudioBus for input and remove the
FromInterleaved here.

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Will do.

+          data, audio_bus_->frames(), audio_params_.bits_per_sample() / 8);
+      audio_level_->Scan(*audio_bus_, audio_bus_->frames());
+
+      // Get current average power level and add it to the log.
+      std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();
+      AddPowerLevelToLog(result.first);
+
+      // Reset the average power level (since we don't log continuously).
+      audio_level_->Reset();

[no longer working on chromium, 2014/05/23 08:29:56]

I am not familiar with AudioPowerMonitor, do you know how this Reset affect the
smoothing factor ? for example, if the frame size in audio_bus is 3ms,
kPowerMeasurementTimeConstantMilliseconds is 10, but you Reset the audio level
after scanning one frame, does this still work?

[henrika (OOO until Aug 14), 2014/05/23 08:42:45]

Yes it works. The smoothing factor controls how fast the filter will detect that
the signal changes but given that we only filter once every other second it
makes no sense to keep the old value in memory. 

Regarding 3ms, let me run some more tests on Mac and check. Let me think some
more on this.

+      last_audio_level_log_time_ = base::TimeTicks::Now();
+    }
+#endif
+
     return;
   }
 
   // TODO(henrika): Investigate if we can avoid the extra copy here.
   // (see http://crbug.com/249316 for details). AFAIK, this scope is only
   // active for WebSpeech clients.
   scoped_ptr<uint8[]> audio_data(new uint8[size]);
   memcpy(audio_data.get(), data, size);
 
   // Ownership of the audio buffer will be with the callback until it is run,
@@ skipping 30 matching lines @@
 
 void AudioInputController::SetDataIsActive(bool enabled) {
   base::subtle::Release_Store(&data_is_active_, enabled);
 }
 
 bool AudioInputController::GetDataIsActive() {
   return (base::subtle::Acquire_Load(&data_is_active_) != false);
 }
 
 }  // namespace media