Adds volume level measurements to the AudioInputController for low-latency clients

media/audio/audio_input_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/audio/audio_input_controller.h"
 
 #include "base/bind.h"
+#include "base/strings/stringprintf.h"
 #include "base/threading/thread_restrictions.h"
+#include "base/time/time.h"
 #include "media/base/limits.h"
 #include "media/base/scoped_histogram_timer.h"
 #include "media/base/user_input_monitor.h"
 
+using base::TimeDelta;
+
 namespace {
 const int kMaxInputChannels = 3;
 
 // TODO(henrika): remove usage of timers and add support for proper
 // notification of when the input device is removed.  This was originally added
 // to resolve http://crbug.com/79936 for Windows platforms.  This then caused
 // breakage (very hard to repro bugs!) on other platforms: See
 // http://crbug.com/226327 and http://crbug.com/230972.
 // See also that the timer has been disabled on Mac now due to
 // crbug.com/357501.
 const int kTimerResetIntervalSeconds = 1;
 // We have received reports that the timer can be too trigger happy on some
 // Mac devices and the initial timer interval has therefore been increased
 // from 1 second to 5 seconds.
 const int kTimerInitialIntervalSeconds = 5;
+
+// Time constant for AudioPowerMonitor.
+// The utilized smoothing factor (alpha) in the exponential filter is given
+// by 1-exp(-1/(fs*ts)), where fs is the sample rate in Hz and ts is the time
+// constant given by |kPowerMeasurementTimeConstantMilliseconds|.
+// Example: fs=44100, ts=10e-3 => alpha~0.022420
+//          fs=44100, ts=20e-3 => alpha~0.165903
+// A large smoothing factor corresponds to a faster filter response to input
+// changes since y(n)=alpha*x(n)+(1-alpha)*y(n-1), where x(n) is the input
+// and y(n) is the output.
+const int kPowerMeasurementTimeConstantMilliseconds = 10;
+
+// Time in seconds between two successive measurements of audio power levels.
+const int kPowerMonitorLogIntervalSeconds = 5;
 }
 
 namespace media {
 
 // static
 AudioInputController::Factory* AudioInputController::factory_ = NULL;
 
 AudioInputController::AudioInputController(EventHandler* handler,
                                            SyncWriter* sync_writer,
                                            UserInputMonitor* user_input_monitor)
@@ skipping 14 matching lines @@
 }
 
 // static
 scoped_refptr<AudioInputController> AudioInputController::Create(
     AudioManager* audio_manager,
     EventHandler* event_handler,
     const AudioParameters& params,
     const std::string& device_id,
     UserInputMonitor* user_input_monitor) {
   DCHECK(audio_manager);
+  DVLOG(1) << "AudioInputController::Create";
 
   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     return NULL;
 
   if (factory_) {
     return factory_->Create(
         audio_manager, event_handler, params, user_input_monitor);
   }
   scoped_refptr<AudioInputController> controller(
       new AudioInputController(event_handler, NULL, user_input_monitor));
@@ skipping 14 matching lines @@
 // static
 scoped_refptr<AudioInputController> AudioInputController::CreateLowLatency(
     AudioManager* audio_manager,
     EventHandler* event_handler,
     const AudioParameters& params,
     const std::string& device_id,
     SyncWriter* sync_writer,
     UserInputMonitor* user_input_monitor) {
   DCHECK(audio_manager);
   DCHECK(sync_writer);
+  DVLOG(1) << "AudioInputController::CreateLowLatency";
 
   if (!params.IsValid() || (params.channels() > kMaxInputChannels))
     return NULL;
 
   // Create the AudioInputController object and ensure that it runs on
   // the audio-manager thread.
   scoped_refptr<AudioInputController> controller(
       new AudioInputController(event_handler, sync_writer, user_input_monitor));
   controller->task_runner_ = audio_manager->GetTaskRunner();
 
@@ skipping 60 matching lines @@
 void AudioInputController::SetAutomaticGainControl(bool enabled) {
   task_runner_->PostTask(FROM_HERE, base::Bind(
       &AudioInputController::DoSetAutomaticGainControl, this, enabled));
 }
 
 void AudioInputController::DoCreate(AudioManager* audio_manager,
                                     const AudioParameters& params,
                                     const std::string& device_id) {
   DCHECK(task_runner_->BelongsToCurrentThread());
   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
+  DVLOG(1) << "AudioInputController::DoCreate";
+
+#if defined(AUDIO_POWER_MONITORING)
+  // Create the audio (power) level meter given the provided audio parameters.
+  // An AudioBus is also needed to wrap the raw data buffer from the native
+  // layer to match AudioPowerMonitor::Scan().
+  // TODO(henrika): Remove use of extra AudioBus. See http://crbug.com/375155.
+  audio_level_.reset(new media::AudioPowerMonitor(
+      params.sample_rate(),
+      TimeDelta::FromMilliseconds(kPowerMeasurementTimeConstantMilliseconds)));
+  audio_bus_ = AudioBus::Create(params);
+  audio_params_ = params;
+#endif
+
   // TODO(miu): See TODO at top of file.  Until that's resolved, assume all
   // platform audio input requires the |no_data_timer_| be used to auto-detect
   // errors.  In reality, probably only Windows needs to be treated as
   // unreliable here.
   DoCreateForStream(audio_manager->MakeAudioInputStream(params, device_id),
                     true);
 }
 
 void AudioInputController::DoCreateForStream(
     AudioInputStream* stream_to_control, bool enable_nodata_timer) {
@@ skipping 73 matching lines @@
 void AudioInputController::DoClose() {
   DCHECK(task_runner_->BelongsToCurrentThread());
   SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
 
   if (state_ == CLOSED)
     return;
 
   // Delete the timer on the same thread that created it.
   no_data_timer_.reset();
 
+#if defined(AUDIO_POWER_MONITORING)
+  std::string log_string("AIC::DoClose: closing input stream");

[no longer working on chromium, 2014/05/26 14:20:16]

nit, const

[henrika (OOO until Aug 14), 2014/05/26 15:01:17]

Removed.

+  if (handler_)
+    handler_->OnLog(this, log_string);
+  if (audio_level_)
+    audio_level_->Reset();
+#endif
+
   DoStopCloseAndClearStream();
   SetDataIsActive(false);
 
   if (SharedMemoryAndSyncSocketMode())
     sync_writer_->Close();
 
   if (user_input_monitor_)
     user_input_monitor_->DisableKeyPressMonitoring();
 
   state_ = CLOSED;
@@ skipping 91 matching lines @@
     key_pressed = current_count != prev_key_down_count_;
     prev_key_down_count_ = current_count;
     DVLOG_IF(6, key_pressed) << "Detected keypress.";
   }
 
   // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
   // Used by all low-latency clients except WebSpeech.
   if (SharedMemoryAndSyncSocketMode()) {
     sync_writer_->Write(data, size, volume, key_pressed);
     sync_writer_->UpdateRecordedBytes(hardware_delay_bytes);
+
+#if defined(AUDIO_POWER_MONITORING)
+    {
+      // Only do power-level measurements if an AudioPowerMonitor object has
+      // been created. Done in DoCreate() but not DoCreateForStream(), hence
+      // logging will mainly be done for WebRTC and WebSpeech clients.
+      base::AutoLock auto_lock(lock_);
+      if (!audio_level_)
+        return;
+    }
+
+    // Perform periodic audio (power) level measurements.
+    if ((base::TimeTicks::Now() - last_audio_level_log_time_).InSeconds() >
+        kPowerMonitorLogIntervalSeconds) {
+      // Wrap data into an AudioBus to match AudioPowerMonitor::Scan.
+      // TODO(henrika): remove this section when capture side uses AudioBus.
+      // See http://crbug.com/375155 for details.
+      audio_bus_->FromInterleaved(
+          data, audio_bus_->frames(), audio_params_.bits_per_sample() / 8);
+      audio_level_->Scan(*audio_bus_, audio_bus_->frames());
+
+      task_runner_->PostTask(
+          FROM_HERE, base::Bind(&AudioInputController::DoLogAudioLevel, this));
+
+      last_audio_level_log_time_ = base::TimeTicks::Now();
+    }
+#endif
+
     return;
   }
 
   // TODO(henrika): Investigate if we can avoid the extra copy here.
   // (see http://crbug.com/249316 for details). AFAIK, this scope is only
   // active for WebSpeech clients.
   scoped_ptr<uint8[]> audio_data(new uint8[size]);
   memcpy(audio_data.get(), data, size);
 
   // Ownership of the audio buffer will be with the callback until it is run,
   // when ownership is passed to the callback function.
   task_runner_->PostTask(FROM_HERE, base::Bind(
       &AudioInputController::DoOnData, this, base::Passed(&audio_data), size));
 }
 
 void AudioInputController::DoOnData(scoped_ptr<uint8[]> data, uint32 size) {
   DCHECK(task_runner_->BelongsToCurrentThread());
   if (handler_)
     handler_->OnData(this, data.get(), size);
 }
 
+void AudioInputController::DoLogAudioLevel() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+
+  // Get current average power level and add it to the log.
+  // Possible range is given by [-inf, 0] dBFS.
+  std::pair<float, bool> result = audio_level_->ReadCurrentPowerAndClip();

[no longer working on chromium, 2014/05/26 14:20:16]

this introduces a race here. audio_level_ is changed/accessed by both the
real-time audio thread and task_runner_ thread.

Instead, you should do all the calculation in the real-time audio thread and
just and post the values to task_runner_ thread.

[henrika (OOO until Aug 14), 2014/05/26 15:01:17]

Will fix.

+  std::string log_string = base::StringPrintf(
+      "AIC::OnData: average audio level=%.2f dBFS", result.first);
+  static const float kSilenceThresholdDBFS = -72.24719896f;
+  if (result.first < kSilenceThresholdDBFS)
+    log_string += " <=> no audio input!";
+  if (handler_)
+    handler_->OnLog(this, log_string);
+
+  // Reset the average power level (since we don't log continuously).
+  audio_level_->Reset();
+}
+
 void AudioInputController::OnError(AudioInputStream* stream) {
   // Handle error on the audio-manager thread.
   task_runner_->PostTask(FROM_HERE, base::Bind(
       &AudioInputController::DoReportError, this));
 }
 
 void AudioInputController::DoStopCloseAndClearStream() {
   DCHECK(task_runner_->BelongsToCurrentThread());
 
   // Allow calling unconditionally and bail if we don't have a stream to close.
   if (stream_ != NULL) {
     stream_->Stop();
     stream_->Close();
     stream_ = NULL;
   }
 
   // The event handler should not be touched after the stream has been closed.
   handler_ = NULL;
 }
 
 void AudioInputController::SetDataIsActive(bool enabled) {
   base::subtle::Release_Store(&data_is_active_, enabled);
 }
 
 bool AudioInputController::GetDataIsActive() {
   return (base::subtle::Acquire_Load(&data_is_active_) != false);
 }
 
 }  // namespace media