MediaStream audio object graph untangling and clean-ups.

content/renderer/media/webrtc_audio_capturer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webrtc_audio_capturer.h"
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/metrics/histogram.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "build/build_config.h"
 #include "content/child/child_process.h"
 #include "content/renderer/media/audio_device_factory.h"
 #include "content/renderer/media/media_stream_audio_processor.h"
 #include "content/renderer/media/media_stream_audio_processor_options.h"
 #include "content/renderer/media/media_stream_audio_source.h"
 #include "content/renderer/media/media_stream_constraints_util.h"
 #include "content/renderer/media/webrtc_audio_device_impl.h"
 #include "content/renderer/media/webrtc_local_audio_track.h"
 #include "content/renderer/media/webrtc_logging.h"
 #include "media/audio/sample_rates.h"
 
 namespace content {
 
-namespace {
-
-// Audio buffer sizes are specified in milliseconds.
-const char kAudioLatency[] = "latencyMs";
-const int kMinAudioLatencyMs = 0;
-const int kMaxAudioLatencyMs = 10000;
-
-// Method to check if any of the data in |audio_source| has energy.
-bool HasDataEnergy(const media::AudioBus& audio_source) {
-  for (int ch = 0; ch < audio_source.channels(); ++ch) {
-    const float* channel_ptr = audio_source.channel(ch);
-    for (int frame = 0; frame < audio_source.frames(); ++frame) {
-      if (channel_ptr[frame] != 0)
-        return true;
-    }
-  }
-
-  // All the data is zero.
-  return false;
-}
-
-}  // namespace
-
 // Reference counted container of WebRtcLocalAudioTrack delegate.
 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
 class WebRtcAudioCapturer::TrackOwner
     : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> {
  public:
   explicit TrackOwner(WebRtcLocalAudioTrack* track)
       : delegate_(track) {}
 
   void Capture(const media::AudioBus& audio_bus,
-               base::TimeTicks estimated_capture_time,
-               bool force_report_nonzero_energy) {
+               base::TimeTicks estimated_capture_time) {
     base::AutoLock lock(lock_);
     if (delegate_) {
-      delegate_->Capture(audio_bus,
-                         estimated_capture_time,
-                         force_report_nonzero_energy);
+      delegate_->Capture(audio_bus, estimated_capture_time);
     }
   }
 
   void OnSetFormat(const media::AudioParameters& params) {
     base::AutoLock lock(lock_);
     if (delegate_)
       delegate_->OnSetFormat(params);
   }
 
-  void SetAudioProcessor(
-      const scoped_refptr<MediaStreamAudioProcessor>& processor) {
-    base::AutoLock lock(lock_);
-    if (delegate_)
-      delegate_->SetAudioProcessor(processor);
-  }
-
   void Reset() {
     base::AutoLock lock(lock_);
     delegate_ = NULL;
   }
 
   void Stop() {
     base::AutoLock lock(lock_);
     DCHECK(delegate_);
 
     // This can be reentrant so reset |delegate_| before calling out.
@@ skipping 25 matching lines @@
   }
 
   // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
   WebRtcLocalAudioTrack* delegate_;
   mutable base::Lock lock_;
 
   DISALLOW_COPY_AND_ASSIGN(TrackOwner);
 };
 
 // static
-scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
+scoped_ptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
     int render_frame_id,
     const StreamDeviceInfo& device_info,
     const blink::WebMediaConstraints& constraints,
     WebRtcAudioDeviceImpl* audio_device,
     MediaStreamAudioSource* audio_source) {
-  scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
-      render_frame_id, device_info, constraints, audio_device, audio_source);
+  scoped_ptr<WebRtcAudioCapturer> capturer(new WebRtcAudioCapturer(
+      render_frame_id, device_info, constraints, audio_device, audio_source));
   if (capturer->Initialize())
     return capturer;
 
   return NULL;
 }
 
 bool WebRtcAudioCapturer::Initialize() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
   WebRtcLogMessage(base::StringPrintf(
@@ skipping 54 matching lines @@
            << device_info_.device.input.sample_rate;
   media::AudioSampleRate asr;
   if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) {
     UMA_HISTOGRAM_ENUMERATION(
         "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
   } else {
     UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
                          device_info_.device.input.sample_rate);
   }
 
-  // Initialize the buffer size to zero, which means it wasn't specified.
-  // If it is out of range, we return it to zero.
-  int buffer_size_ms = 0;
-  int buffer_size_samples = 0;
-  GetConstraintValueAsInteger(constraints_, kAudioLatency, &buffer_size_ms);
-  if (buffer_size_ms < kMinAudioLatencyMs ||
-      buffer_size_ms > kMaxAudioLatencyMs) {
-    DVLOG(1) << "Ignoring out of range buffer size " << buffer_size_ms;
-  } else {
-    buffer_size_samples =
-        device_info_.device.input.sample_rate * buffer_size_ms / 1000;
-  }
-  DVLOG_IF(1, buffer_size_samples > 0)
-      << "Custom audio buffer size: " << buffer_size_samples << " samples";
-
   // Create and configure the default audio capturing source.
   SetCapturerSourceInternal(
-      AudioDeviceFactory::NewInputDevice(render_frame_id_),
-      channel_layout,
-      device_info_.device.input.sample_rate,
-      buffer_size_samples);
+      AudioDeviceFactory::NewInputDevice(render_frame_id_), channel_layout,
+      device_info_.device.input.sample_rate);
 
   // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
   // information from the capturer.
   if (audio_device_)
     audio_device_->AddAudioCapturer(this);
 
   return true;
 }
 
 WebRtcAudioCapturer::WebRtcAudioCapturer(
     int render_frame_id,
     const StreamDeviceInfo& device_info,
     const blink::WebMediaConstraints& constraints,
     WebRtcAudioDeviceImpl* audio_device,
     MediaStreamAudioSource* audio_source)
     : constraints_(constraints),
       audio_processor_(new rtc::RefCountedObject<MediaStreamAudioProcessor>(
           constraints,
           device_info.device.input,
           audio_device)),
       running_(false),
       render_frame_id_(render_frame_id),
       device_info_(device_info),
       volume_(0),
       peer_connection_mode_(false),
       audio_device_(audio_device),
-      audio_source_(audio_source) {
+      audio_source_(audio_source),
+      weak_factory_(this) {
   DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
 }
 
 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(tracks_.IsEmpty());
   DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
   Stop();
 }
 
 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
+  DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(track);
   DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
 
+  track->AddStopObserver(base::Bind(&WebRtcAudioCapturer::RemoveTrack,
+                                    weak_factory_.GetWeakPtr(), track));
+  track->SetLevel(level_calculator_.level());
+
+  // The track only grabs stats from the audio processor.  Stats are only
+  // available if audio processing is turned on.  Therefore, only provide the
+  // track a reference if audio processing is turned on.
+  if (audio_processor_->has_audio_processing())
+    track->SetAudioProcessor(audio_processor_);
+
   {
     base::AutoLock auto_lock(lock_);
     // Verify that |track| is not already added to the list.
     DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track)));
 
     // Add with a tag, so we remember to call OnSetFormat() on the new
     // track.
     scoped_refptr<TrackOwner> track_owner(new TrackOwner(track));
     tracks_.AddAndTag(track_owner.get());
   }
@@ skipping 23 matching lines @@
     // we have to call StopSource on the MediaStreamSource. This will call
     // MediaStreamAudioSource::DoStopSource which in turn call
     // WebRtcAudioCapturerer::Stop();
     audio_source_->StopSource();
   }
 }
 
 void WebRtcAudioCapturer::SetCapturerSourceInternal(
     const scoped_refptr<media::AudioCapturerSource>& source,
     media::ChannelLayout channel_layout,
-    int sample_rate,
-    int buffer_size) {
+    int sample_rate) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
            << "sample_rate=" << sample_rate << ")";
   scoped_refptr<media::AudioCapturerSource> old_source;
   {
     base::AutoLock auto_lock(lock_);
     if (source_.get() == source.get())
       return;
 
     source_.swap(old_source);
     source_ = source;
 
     // Reset the flag to allow starting the new source.
     running_ = false;
   }
 
   DVLOG(1) << "Switching to a new capture source.";
   if (old_source.get())
     old_source->Stop();
 
-  // If the buffer size is zero, it has not been specified.
-  // We either default to 10ms, or use the hardware buffer size.
-  if (buffer_size == 0)
-    buffer_size = GetBufferSize(sample_rate);
-
   // Dispatch the new parameters both to the sink(s) and to the new source,
   // also apply the new |constraints|.
   // The idea is to get rid of any dependency of the microphone parameters
   // which would normally be used by default.
   // bits_per_sample is always 16 for now.
   media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
-                                channel_layout, sample_rate, 16, buffer_size);
+                                channel_layout, sample_rate, 16,
+                                GetBufferSize(sample_rate));
   params.set_effects(device_info_.device.input.effects);
+  DCHECK(params.IsValid());
 
   {
     base::AutoLock auto_lock(lock_);
     // Notify the |audio_processor_| of the new format.
     audio_processor_->OnCaptureFormatChanged(params);
 
     // Notify all tracks about the new format.
     tracks_.TagAll();
   }
 
   if (source.get())
-    source->Initialize(params, this, session_id());
+    source->Initialize(params, this, device_info_.session_id);
 
   Start();
 }
 
 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "EnablePeerConnectionMode";
   // Do nothing if the peer connection mode has been enabled.
   if (peer_connection_mode_)
     return;
@@ skipping 15 matching lines @@
   // Do nothing if the current buffer size is the WebRtc native buffer size.
   if (GetBufferSize(input_params.sample_rate()) ==
           input_params.frames_per_buffer()) {
     return;
   }
 
   // Create a new audio stream as source which will open the hardware using
   // WebRtc native buffer size.
   SetCapturerSourceInternal(AudioDeviceFactory::NewInputDevice(render_frame_id),
                             input_params.channel_layout(),
-                            input_params.sample_rate(),
-                            0);
+                            input_params.sample_rate());
 }
 
 void WebRtcAudioCapturer::Start() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "WebRtcAudioCapturer::Start()";
   base::AutoLock auto_lock(lock_);
   if (running_ || !source_.get())
     return;
 
   // Start the data source, i.e., start capturing data from the current source.
@@ skipping 16 matching lines @@
     source = source_;
     tracks = tracks_.Items();
     tracks_.Clear();
     running_ = false;
   }
 
   // Remove the capturer object from the WebRtcAudioDeviceImpl.
   if (audio_device_)
     audio_device_->RemoveAudioCapturer(this);
 
+  // Invalidate the weak pointers since we don't need the tracks to call our
+  // RemoveTrack() method when their Stop() method is called.
+  weak_factory_.InvalidateWeakPtrs();
+
   for (TrackList::ItemList::const_iterator it = tracks.begin();
        it != tracks.end();
        ++it) {
     (*it)->Stop();
   }
 
   if (source.get())
     source->Stop();
 
   // Stop the audio processor to avoid feeding render data into the processor.
@@ skipping 56 matching lines @@
 
     // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
     // The volume can be higher than 255 on Linux, and it will be cropped to
     // 255 since AGC does not allow values out of range.
     volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
     current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
     tracks = tracks_.Items();
     tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
   }
 
   DCHECK(audio_processor_->InputFormat().IsValid());

[o1ka, 2016/02/29 14:28:04]

I got lost in thread safety of |audio_processor_| here. Should we either assure
that we are on the valid thread, or use the lock to access InputFormat(), as it
is done in other places?

[miu, 2016/03/01 09:43:55]

Agree it's confusing.  Here's how I understand it:

1. The call to audio_processor_->OnCaptureFormatChanged() is what would mutate
the value InputFormat() reads.  This OnCaptureFromatChanged() call only occurs
on the main thread AND only at a time when we know the audio thread will not be
in the middle of executing the code in Capture().  (We know the audio thread
will not be calling Capture() because the "source" is stopped.)

2. The locking is needed in GetInputFormat() because it's the WebRTC signaling
thread attempting to read from something that could be mutated at any time on
the main thread.

So, because of #1, locking shouldn't be needed here at these lines of code.

[o1ka, 2016/03/01 14:18:59]

Great! Can we have this in the comment? It would be invaluable for unprepared
readers like me. (Though I've got a feeling it can be refactored to be more
clear and straightforward at some point, but I guess it's out of scope of this
change)

[miu, 2016/03/02 01:12:50]

Done.  And, yes my upcoming changes will clear this up a lot!  :)

[o1ka, 2016/03/02 16:31:14]

Great,thanks!

   DCHECK_EQ(audio_source->channels(),
             audio_processor_->InputFormat().channels());
   DCHECK_EQ(audio_source->frames(),
             audio_processor_->InputFormat().frames_per_buffer());
 
   // Notify the tracks on when the format changes. This will do nothing if
   // |tracks_to_notify_format| is empty.
   const media::AudioParameters& output_params =
       audio_processor_->OutputFormat();

[o1ka, 2016/02/29 14:28:04]

Should we be on the valid thread, see Line 427?

[miu, 2016/03/01 09:43:55]

No.  This is another case of #1 (in my last comment).

[o1ka, 2016/03/01 14:18:59]

Could you make a comment on this? Would be very helpful.

[miu, 2016/03/02 01:12:50]

Done.  I also "grouped" this code with the DCHECKs above since they're all
related in this same way.

-  for (const auto& track : tracks_to_notify_format) {
+  for (const auto& track : tracks_to_notify_format)
     track->OnSetFormat(output_params);
-    track->SetAudioProcessor(audio_processor_);
-  }
 
-  // Figure out if the pre-processed data has any energy or not, the
-  // information will be passed to the track to force the calculator
-  // to report energy in case the post-processed data is zeroed by the audio
-  // processing.
-  const bool force_report_nonzero_energy = HasDataEnergy(*audio_source);
+  // Figure out if the pre-processed data has any energy or not. This
+  // information will be passed to the level calculator to force it to report
+  // energy in case the post-processed data is zeroed by the audio processing.
+  const bool force_report_nonzero_energy = !audio_source->AreFramesZero();
 
   // Push the data to the processor for processing.
   audio_processor_->PushCaptureData(
       *audio_source,
       base::TimeDelta::FromMilliseconds(audio_delay_milliseconds));
 
   // Process and consume the data in the processor until there is not enough
   // data in the processor.
   media::AudioBus* processed_data = nullptr;
   base::TimeDelta processed_data_audio_delay;
   int new_volume = 0;
   while (audio_processor_->ProcessAndConsumeData(
              current_volume, key_pressed,
              &processed_data, &processed_data_audio_delay, &new_volume)) {
     DCHECK(processed_data);
+
+    level_calculator_.Calculate(*processed_data, force_report_nonzero_energy);
+
     const base::TimeTicks processed_data_capture_time =
         reference_clock_snapshot - processed_data_audio_delay;
-    for (const auto& track : tracks) {
-      track->Capture(*processed_data,
-                     processed_data_capture_time,
-                     force_report_nonzero_energy);
-    }
+    for (const auto& track : tracks)
+      track->Capture(*processed_data, processed_data_capture_time);
 
     if (new_volume) {
       SetVolume(new_volume);
 
       // Update the |current_volume| to avoid passing the old volume to AGC.
       current_volume = new_volume;
     }
   }
 }
 
 void WebRtcAudioCapturer::OnCaptureError(const std::string& message) {
   WebRtcLogMessage("WAC::OnCaptureError: " + message);
 }
 
-media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const {
+media::AudioParameters WebRtcAudioCapturer::GetInputFormat() const {
   base::AutoLock auto_lock(lock_);
-  return audio_processor_.get() ? audio_processor_->InputFormat()
-                                : media::AudioParameters();
-}
-
-bool WebRtcAudioCapturer::GetPairedOutputParameters(
-    int* session_id,
-    int* output_sample_rate,
-    int* output_frames_per_buffer) const {
-  // Don't set output parameters unless all of them are valid.
-  if (device_info_.session_id <= 0 ||
-      !device_info_.device.matched_output.sample_rate ||
-      !device_info_.device.matched_output.frames_per_buffer)
-    return false;
-
-  *session_id = device_info_.session_id;
-  *output_sample_rate = device_info_.device.matched_output.sample_rate;
-  *output_frames_per_buffer =
-      device_info_.device.matched_output.frames_per_buffer;
-
-  return true;
+  return audio_processor_->InputFormat();
 }
 
 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
   DCHECK(thread_checker_.CalledOnValidThread());
 #if defined(OS_ANDROID)
   // TODO(henrika): Tune and adjust buffer size on Android.
   return (2 * sample_rate / 100);
 #endif
 
   // PeerConnection is running at a buffer size of 10ms data. A multiple of
@@ skipping 12 matching lines @@
     return hardware_buffer_size;
   }
 
   return (sample_rate / 100);
 }
 
 void WebRtcAudioCapturer::SetCapturerSource(
     const scoped_refptr<media::AudioCapturerSource>& source,
     media::AudioParameters params) {
   // Create a new audio stream as source which uses the new source.
-  SetCapturerSourceInternal(source,
-                            params.channel_layout(),
-                            params.sample_rate(),
-                            0);
+  SetCapturerSourceInternal(source, params.channel_layout(),
+                            params.sample_rate());
 }
 
 }  // namespace content