MediaStream audio object graph untangling and clean-ups.

content/renderer/media/webrtc_audio_capturer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webrtc_audio_capturer.h"
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "build/build_config.h"
 #include "content/child/child_process.h"
 #include "content/renderer/media/audio_device_factory.h"
 #include "content/renderer/media/media_stream_audio_processor.h"
 #include "content/renderer/media/media_stream_audio_processor_options.h"
 #include "content/renderer/media/media_stream_audio_source.h"
 #include "content/renderer/media/media_stream_constraints_util.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
 #include "content/renderer/media/webrtc_local_audio_track.h"
 #include "content/renderer/media/webrtc_logging.h"
 #include "media/audio/sample_rates.h"
 
 namespace content {
 
-namespace {
-
-// Audio buffer sizes are specified in milliseconds.
-const char kAudioLatency[] = "latencyMs";
-const int kMinAudioLatencyMs = 0;
-const int kMaxAudioLatencyMs = 10000;
-
-// Method to check if any of the data in |audio_source| has energy.
-bool HasDataEnergy(const media::AudioBus& audio_source) {
-  for (int ch = 0; ch < audio_source.channels(); ++ch) {
-    const float* channel_ptr = audio_source.channel(ch);
-    for (int frame = 0; frame < audio_source.frames(); ++frame) {
-      if (channel_ptr[frame] != 0)
-        return true;
-    }
-  }
-
-  // All the data is zero.
-  return false;
-}
-
-}  // namespace
-
 // Reference counted container of WebRtcLocalAudioTrack delegate.
 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
 class WebRtcAudioCapturer::TrackOwner
     : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> {
  public:
   explicit TrackOwner(WebRtcLocalAudioTrack* track)
       : delegate_(track) {}
 
   void Capture(const media::AudioBus& audio_bus,
-               base::TimeTicks estimated_capture_time,
-               bool force_report_nonzero_energy) {
+               base::TimeTicks estimated_capture_time) {
     base::AutoLock lock(lock_);
     if (delegate_) {
-      delegate_->Capture(audio_bus,
-                         estimated_capture_time,
-                         force_report_nonzero_energy);
+      delegate_->Capture(audio_bus, estimated_capture_time);
     }
   }
 
   void OnSetFormat(const media::AudioParameters& params) {
     base::AutoLock lock(lock_);
     if (delegate_)
       delegate_->OnSetFormat(params);
   }
 
-  void SetAudioProcessor(
-      const scoped_refptr<MediaStreamAudioProcessor>& processor) {
-    base::AutoLock lock(lock_);
-    if (delegate_)
-      delegate_->SetAudioProcessor(processor);
-  }
-
   void Reset() {
     base::AutoLock lock(lock_);
     delegate_ = NULL;
   }
 
   void Stop() {
     base::AutoLock lock(lock_);
     DCHECK(delegate_);
 
     // This can be reentrant so reset |delegate_| before calling out.
@@ skipping 25 matching lines @@
   }
 
   // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
   WebRtcLocalAudioTrack* delegate_;
   mutable base::Lock lock_;
 
   DISALLOW_COPY_AND_ASSIGN(TrackOwner);
 };
 
 // static
-scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
+scoped_ptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
     int render_frame_id,
     const StreamDeviceInfo& device_info,
     const blink::WebMediaConstraints& constraints,
     WebRtcAudioDeviceImpl* audio_device,
     MediaStreamAudioSource* audio_source) {
-  scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
-      render_frame_id, device_info, constraints, audio_device, audio_source);
+  scoped_ptr<WebRtcAudioCapturer> capturer(new WebRtcAudioCapturer(
+      render_frame_id, device_info, constraints, audio_device, audio_source));
   if (capturer->Initialize())
     return capturer;
 
   return NULL;
 }
 
 bool WebRtcAudioCapturer::Initialize() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
   WebRtcLogMessage(base::StringPrintf(
@@ skipping 54 matching lines @@
            << device_info_.device.input.sample_rate;
   media::AudioSampleRate asr;
   if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) {
     UMA_HISTOGRAM_ENUMERATION(
         "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
   } else {
     UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
                          device_info_.device.input.sample_rate);
   }
 
-  // Initialize the buffer size to zero, which means it wasn't specified.
-  // If it is out of range, we return it to zero.
-  int buffer_size_ms = 0;
-  int buffer_size_samples = 0;
-  GetConstraintValueAsInteger(constraints_, kAudioLatency, &buffer_size_ms);
-  if (buffer_size_ms < kMinAudioLatencyMs ||
-      buffer_size_ms > kMaxAudioLatencyMs) {
-    DVLOG(1) << "Ignoring out of range buffer size " << buffer_size_ms;
-  } else {
-    buffer_size_samples =
-        device_info_.device.input.sample_rate * buffer_size_ms / 1000;
-  }
-  DVLOG_IF(1, buffer_size_samples > 0)
-      << "Custom audio buffer size: " << buffer_size_samples << " samples";
-
   // Create and configure the default audio capturing source.
   SetCapturerSourceInternal(
-      AudioDeviceFactory::NewInputDevice(render_frame_id_),
-      channel_layout,
-      device_info_.device.input.sample_rate,
-      buffer_size_samples);
+      AudioDeviceFactory::NewInputDevice(render_frame_id_), channel_layout,
+      device_info_.device.input.sample_rate);
 
   // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
   // information from the capturer.
   if (audio_device_)
     audio_device_->AddAudioCapturer(this);
 
   return true;
 }
 
 WebRtcAudioCapturer::WebRtcAudioCapturer(
@@ skipping 18 matching lines @@
 }
 
 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(tracks_.IsEmpty());
   DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
   Stop();
 }
 
 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
+  DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(track);
   DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
 
+  track->SetLevel(level_calculator_.level());
+
+  // The track only grabs stats from the audio processor.  Stats are only
+  // available if audio processing is turned on.  Therefore, only provide the
+  // track a reference if audio processing is turned on.
+  if (audio_processor_->has_audio_processing())
+    track->SetAudioProcessor(audio_processor_);
+
   {
     base::AutoLock auto_lock(lock_);
     // Verify that |track| is not already added to the list.
     DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track)));
 
     // Add with a tag, so we remember to call OnSetFormat() on the new
     // track.
     scoped_refptr<TrackOwner> track_owner(new TrackOwner(track));
     tracks_.AddAndTag(track_owner.get());
   }
@@ skipping 23 matching lines @@
     // we have to call StopSource on the MediaStreamSource. This will call
     // MediaStreamAudioSource::DoStopSource which in turn call
     // WebRtcAudioCapturerer::Stop();
     audio_source_->StopSource();
   }
 }
 
 void WebRtcAudioCapturer::SetCapturerSourceInternal(
     const scoped_refptr<media::AudioCapturerSource>& source,
     media::ChannelLayout channel_layout,
-    int sample_rate,
-    int buffer_size) {
+    int sample_rate) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
            << "sample_rate=" << sample_rate << ")";
   scoped_refptr<media::AudioCapturerSource> old_source;
   {
     base::AutoLock auto_lock(lock_);
     if (source_.get() == source.get())
       return;
 
     source_.swap(old_source);
     source_ = source;
 
     // Reset the flag to allow starting the new source.
     running_ = false;
   }
 
   DVLOG(1) << "Switching to a new capture source.";
   if (old_source.get())
     old_source->Stop();
 
-  // If the buffer size is zero, it has not been specified.
-  // We either default to 10ms, or use the hardware buffer size.
-  if (buffer_size == 0)
-    buffer_size = GetBufferSize(sample_rate);
-
   // Dispatch the new parameters both to the sink(s) and to the new source,
   // also apply the new |constraints|.
   // The idea is to get rid of any dependency of the microphone parameters
   // which would normally be used by default.
   // bits_per_sample is always 16 for now.
   media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
-                                channel_layout, sample_rate, 16, buffer_size);
+                                channel_layout, sample_rate, 16,
+                                GetBufferSize(sample_rate));
   params.set_effects(device_info_.device.input.effects);
+  DCHECK(params.IsValid());
 
   {
     base::AutoLock auto_lock(lock_);
-    // Notify the |audio_processor_| of the new format.
+
+    // Notify the |audio_processor_| of the new format.  We're doing this while
+    // the lock is held only because the signaling thread might be calling
+    // GetInputFormat().  Simultaneous reads from the audio thread are NOT the
+    // concern here since the source is currently stopped (i.e., no audio
+    // capture calls can be executing).
     audio_processor_->OnCaptureFormatChanged(params);
 
     // Notify all tracks about the new format.
     tracks_.TagAll();
   }
 
   if (source.get())
-    source->Initialize(params, this, session_id());
+    source->Initialize(params, this, device_info_.session_id);
 
   Start();
 }
 
 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "EnablePeerConnectionMode";
   // Do nothing if the peer connection mode has been enabled.
   if (peer_connection_mode_)
     return;
@@ skipping 15 matching lines @@
   // Do nothing if the current buffer size is the WebRtc native buffer size.
   if (GetBufferSize(input_params.sample_rate()) ==
           input_params.frames_per_buffer()) {
     return;
   }
 
   // Create a new audio stream as source which will open the hardware using
   // WebRtc native buffer size.
   SetCapturerSourceInternal(AudioDeviceFactory::NewInputDevice(render_frame_id),
                             input_params.channel_layout(),
-                            input_params.sample_rate(),
-                            0);
+                            input_params.sample_rate());
 }
 
 void WebRtcAudioCapturer::Start() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "WebRtcAudioCapturer::Start()";
   base::AutoLock auto_lock(lock_);
   if (running_ || !source_.get())
     return;
 
   // Start the data source, i.e., start capturing data from the current source.
@@ skipping 92 matching lines @@
 
     // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
     // The volume can be higher than 255 on Linux, and it will be cropped to
     // 255 since AGC does not allow values out of range.
     volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
     current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
     tracks = tracks_.Items();
     tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
   }
 
+  // Sanity-check the input audio format in debug builds.  Then, notify the
+  // tracks if the format has changed.
+  //
+  // Locking is not needed here to read the audio input/output parameters
+  // because the audio processor format changes only occur while audio capture
+  // is stopped.
   DCHECK(audio_processor_->InputFormat().IsValid());
   DCHECK_EQ(audio_source->channels(),
             audio_processor_->InputFormat().channels());
   DCHECK_EQ(audio_source->frames(),
             audio_processor_->InputFormat().frames_per_buffer());
-
-  // Notify the tracks on when the format changes. This will do nothing if
-  // |tracks_to_notify_format| is empty.
-  const media::AudioParameters& output_params =
-      audio_processor_->OutputFormat();
-  for (const auto& track : tracks_to_notify_format) {
-    track->OnSetFormat(output_params);
-    track->SetAudioProcessor(audio_processor_);
+  if (!tracks_to_notify_format.empty()) {
+    const media::AudioParameters& output_params =
+        audio_processor_->OutputFormat();
+    for (const auto& track : tracks_to_notify_format)
+      track->OnSetFormat(output_params);
   }
 
-  // Figure out if the pre-processed data has any energy or not, the
-  // information will be passed to the track to force the calculator
-  // to report energy in case the post-processed data is zeroed by the audio
-  // processing.
-  const bool force_report_nonzero_energy = HasDataEnergy(*audio_source);
+  // Figure out if the pre-processed data has any energy or not. This
+  // information will be passed to the level calculator to force it to report
+  // energy in case the post-processed data is zeroed by the audio processing.
+  const bool force_report_nonzero_energy = !audio_source->AreFramesZero();
 
   // Push the data to the processor for processing.
   audio_processor_->PushCaptureData(
       *audio_source,
       base::TimeDelta::FromMilliseconds(audio_delay_milliseconds));
 
   // Process and consume the data in the processor until there is not enough
   // data in the processor.
   media::AudioBus* processed_data = nullptr;
   base::TimeDelta processed_data_audio_delay;
   int new_volume = 0;
   while (audio_processor_->ProcessAndConsumeData(
              current_volume, key_pressed,
              &processed_data, &processed_data_audio_delay, &new_volume)) {
     DCHECK(processed_data);
+
+    level_calculator_.Calculate(*processed_data, force_report_nonzero_energy);
+
     const base::TimeTicks processed_data_capture_time =
         reference_clock_snapshot - processed_data_audio_delay;
-    for (const auto& track : tracks) {
-      track->Capture(*processed_data,
-                     processed_data_capture_time,
-                     force_report_nonzero_energy);
-    }
+    for (const auto& track : tracks)
+      track->Capture(*processed_data, processed_data_capture_time);
 
     if (new_volume) {
       SetVolume(new_volume);
 
       // Update the |current_volume| to avoid passing the old volume to AGC.
       current_volume = new_volume;
     }
   }
 }
 
 void WebRtcAudioCapturer::OnCaptureError(const std::string& message) {
   WebRtcLogMessage("WAC::OnCaptureError: " + message);
 }
 
-media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const {
+media::AudioParameters WebRtcAudioCapturer::GetInputFormat() const {
   base::AutoLock auto_lock(lock_);
-  return audio_processor_.get() ? audio_processor_->InputFormat()
-                                : media::AudioParameters();
-}
-
-bool WebRtcAudioCapturer::GetPairedOutputParameters(
-    int* session_id,
-    int* output_sample_rate,
-    int* output_frames_per_buffer) const {
-  // Don't set output parameters unless all of them are valid.
-  if (device_info_.session_id <= 0 ||
-      !device_info_.device.matched_output.sample_rate ||
-      !device_info_.device.matched_output.frames_per_buffer)
-    return false;
-
-  *session_id = device_info_.session_id;
-  *output_sample_rate = device_info_.device.matched_output.sample_rate;
-  *output_frames_per_buffer =
-      device_info_.device.matched_output.frames_per_buffer;
-
-  return true;
+  return audio_processor_->InputFormat();
 }
 
 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
   DCHECK(thread_checker_.CalledOnValidThread());
 #if defined(OS_ANDROID)
   // TODO(henrika): Tune and adjust buffer size on Android.
   return (2 * sample_rate / 100);
 #endif
 
   // PeerConnection is running at a buffer size of 10ms data. A multiple of
@@ skipping 12 matching lines @@
     return hardware_buffer_size;
   }
 
   return (sample_rate / 100);
 }
 
 void WebRtcAudioCapturer::SetCapturerSource(
     const scoped_refptr<media::AudioCapturerSource>& source,
     media::AudioParameters params) {
   // Create a new audio stream as source which uses the new source.
-  SetCapturerSourceInternal(source,
-                            params.channel_layout(),
-                            params.sample_rate(),
-                            0);
+  SetCapturerSourceInternal(source, params.channel_layout(),
+                            params.sample_rate());
 }
 
 }  // namespace content