Feed the render data to MediaStreamAudioProcessor and used AudioBus in render callback

content/renderer/media/media_stream_audio_processor.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_audio_processor.h"
 
 #include "base/command_line.h"
 #include "base/debug/trace_event.h"
 #include "content/public/common/content_switches.h"
 #include "content/renderer/media/media_stream_audio_processor_options.h"
@@ skipping 123 matching lines @@
   // TODO(xians): consider using SincResampler to save some memcpy.
   // Handles mixing and resampling between input and output parameters.
   media::AudioConverter audio_converter_;
   scoped_ptr<media::AudioBus> audio_wrapper_;
   scoped_ptr<media::AudioFifo> fifo_;
 };
 
 MediaStreamAudioProcessor::MediaStreamAudioProcessor(
     const media::AudioParameters& source_params,
     const blink::WebMediaConstraints& constraints,
-    int effects)
+    int effects,
+    WebRtcPlayoutDataSource* playout_data_source)
     : render_delay_ms_(0),
+      playout_data_source_(playout_data_source),
       audio_mirroring_(false) {
   capture_thread_checker_.DetachFromThread();
   render_thread_checker_.DetachFromThread();
   InitializeAudioProcessingModule(constraints, effects);
   InitializeCaptureConverter(source_params);
 }
 
 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() {
-  DCHECK(main_thread_checker_.CalledOnValidThread());

[tommi (sloooow) - chröme, 2014/02/06 16:47:01]

why removing?
(it seems like a good thing to make sure the object is destructed on the same
thread it was constructed on)

[no longer working on chromium, 2014/02/06 17:20:14]

MediaStreamAudioProcessor is reference counted, and the audio thread can grab a
reference and release it there.

Normally this is not a problem since we usually stop the audio thread before we
destuct the MediaStreamAudioProcessor. But I am afraid we don't really guarantee
it.
I removed this thread check since it does not matter in which thread this
destruction happens.

[tommi (sloooow) - chröme, 2014/02/06 17:58:44]

Can we move construction to the audio thread then as well?

[no longer working on chromium, 2014/02/17 13:27:28]

WebRtcAudioCapturer today creates/deletes its resources in the main render
thread since AudioInputDevice needs to be created/deleted on the main render
thread. And I would like to do this to all the resources instead of splitting
them into parts.

I removed the thread check here since I thought it was not a problem to have the
destruction happens in another thread. But since this introduces concerns, I
added it back.

   StopAudioProcessing();
 }
 
 void MediaStreamAudioProcessor::PushCaptureData(media::AudioBus* audio_source) {
   DCHECK(capture_thread_checker_.CalledOnValidThread());
   capture_converter_->Push(audio_source);
 }
 
-void MediaStreamAudioProcessor::PushRenderData(
-    const int16* render_audio, int sample_rate, int number_of_channels,
-    int number_of_frames, base::TimeDelta render_delay) {
-  DCHECK(render_thread_checker_.CalledOnValidThread());
-
-  // Return immediately if the echo cancellation is off.
-  if (!audio_processing_ ||
-      !audio_processing_->echo_cancellation()->is_enabled()) {
-    return;
-  }
-
-  TRACE_EVENT0("audio",
-               "MediaStreamAudioProcessor::FeedRenderDataToAudioProcessing");
-  int64 new_render_delay_ms = render_delay.InMilliseconds();
-  DCHECK_LT(new_render_delay_ms,
-            std::numeric_limits<base::subtle::Atomic32>::max());
-  base::subtle::Release_Store(&render_delay_ms_, new_render_delay_ms);
-
-  InitializeRenderConverterIfNeeded(sample_rate, number_of_channels,
-                                    number_of_frames);
-
-  // TODO(xians): Avoid this extra interleave/deinterleave.
-  render_data_bus_->FromInterleaved(render_audio,
-                                    render_data_bus_->frames(),
-                                    sizeof(render_audio[0]));
-  render_converter_->Push(render_data_bus_.get());
-  while (render_converter_->Convert(&render_frame_))
-    audio_processing_->AnalyzeReverseStream(&render_frame_);
-}
-
 bool MediaStreamAudioProcessor::ProcessAndConsumeData(
     base::TimeDelta capture_delay, int volume, bool key_pressed,
     int* new_volume, int16** out) {
   DCHECK(capture_thread_checker_.CalledOnValidThread());
-  TRACE_EVENT0("audio",
-               "MediaStreamAudioProcessor::ProcessAndConsumeData");
+  TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData");
 
   if (!capture_converter_->Convert(&capture_frame_))
     return false;
 
   *new_volume = ProcessData(&capture_frame_, capture_delay, volume,
                             key_pressed);
   *out = capture_frame_.data_;
 
   return true;
 }
 
 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const {
   return capture_converter_->source_parameters();
 }
 
 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const {
   return capture_converter_->sink_parameters();
 }
 
+void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
+                                              int sample_rate,
+                                              int audio_delay_milliseconds) {
+  DCHECK(render_thread_checker_.CalledOnValidThread());
+  DCHECK(audio_processing_->echo_cancellation()->is_enabled());

[tommi (sloooow) - chröme, 2014/02/06 16:47:01]

is this correct?  will we only ever get this callback if aec is enabled?  What
if only agc is turned on etc?

previously there was a check for it that handled it as a supported case but
having a DCHECK means this is a state that should be impossible for us to reach.

[no longer working on chromium, 2014/02/06 17:20:14]

the playout data is only needed when AEC is enabled, and it does nothing to AGC
or any other component.
In Line 264, we add this processor as sink for the playout data only when the
AEC is enabled.

[tommi (sloooow) - chröme, 2014/02/06 17:58:44]

OK, cool.

+
+  TRACE_EVENT0("audio", "MediaStreamAudioProcessor::OnPlayoutData");
+  DCHECK_LT(audio_delay_milliseconds,
+            std::numeric_limits<base::subtle::Atomic32>::max());
+  base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds);
+
+  InitializeRenderConverterIfNeeded(sample_rate, audio_bus->channels(),
+                                    audio_bus->frames());
+
+  render_converter_->Push(audio_bus);
+  while (render_converter_->Convert(&render_frame_))
+    audio_processing_->AnalyzeReverseStream(&render_frame_);
+}
+
 void MediaStreamAudioProcessor::InitializeAudioProcessingModule(
     const blink::WebMediaConstraints& constraints, int effects) {
   DCHECK(!audio_processing_);
   if (!CommandLine::ForCurrentProcess()->HasSwitch(
           switches::kEnableAudioTrackProcessing)) {
     return;
   }
 
   RTCMediaConstraints native_constraints(constraints);
   ApplyFixedAudioConstraints(&native_constraints);
@@ skipping 41 matching lines @@
   }
 
   // Create and configure the webrtc::AudioProcessing.
   audio_processing_.reset(webrtc::AudioProcessing::Create(0));
 
   // Enable the audio processing components.
   if (enable_aec) {
     EnableEchoCancellation(audio_processing_.get());
     if (enable_experimental_aec)
       EnableExperimentalEchoCancellation(audio_processing_.get());
+
+    if (playout_data_source_)
+      playout_data_source_->AddPlayoutSink(this);
   }
 
   if (enable_ns)
     EnableNoiseSuppression(audio_processing_.get());
 
   if (enable_high_pass_filter)
     EnableHighPassFilter(audio_processing_.get());
 
   if (enable_typing_detection)
     EnableTypingDetection(audio_processing_.get());
@@ skipping 112 matching lines @@
   // Return 0 if the volume has not been changed, otherwise return the new
   // volume.
   return (agc->stream_analog_level() == volume) ?
       0 : agc->stream_analog_level();
 }
 
 void MediaStreamAudioProcessor::StopAudioProcessing() {
   if (!audio_processing_.get())
     return;
 
+  if (playout_data_source_)
+    playout_data_source_->RemovePlayoutSink(this);
+
   audio_processing_.reset();
 }
 
 }  // namespace content