Added an "enable-audio-processor" flag and WebRtcAudioProcessor class

content/renderer/media/webrtc_audio_processor.cc

+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/webrtc_audio_processor.h"
+
+#include "base/command_line.h"
+#include "base/debug/trace_event.h"
+#include "content/public/common/content_switches.h"
+#include "content/renderer/media/webrtc_audio_processor_util.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_converter.h"
+#include "media/base/audio_fifo.h"
+#include "media/base/channel_layout.h"
+
+namespace content {
+
+namespace {
+
+using webrtc::AudioProcessing;
+using webrtc::MediaConstraintsInterface;
+
+#if defined(ANDROID)
+const int kAudioProcessingSampleRate = 16000;
+#else
+const int kAudioProcessingSampleRate = 32000;
+#endif
+const int kAudioProcessingNumberOfChannel = 1;
+
+const int kMaxNumberOfBuffersInFifo = 2;
+
+}  // namespace
+
+class WebRtcAudioProcessor::WebRtcAudioConverter
+    : public media::AudioConverter::InputCallback {
+ public:
+  WebRtcAudioConverter(const media::AudioParameters& source_params,
+                       const media::AudioParameters& sink_params)
+     : source_params_(source_params),
+       sink_params_(sink_params),
+       audio_converter_(source_params, sink_params_, false) {
+    audio_converter_.AddInput(this);
+    // Create and initialize audio fifo and audio bus wrapper.
+    // The size of the FIFO should be at least twice of the source buffer size
+    // or twice of the sink buffer size.
+    int buffer_size = std::max(
+        kMaxNumberOfBuffersInFifo * source_params_.frames_per_buffer(),
+        kMaxNumberOfBuffersInFifo * sink_params_.frames_per_buffer());
+    fifo_.reset(new media::AudioFifo(source_params_.channels(), buffer_size));
+    // TODO(xians): Use CreateWrapper to save one memcpy.
+    audio_wrapper_ = media::AudioBus::Create(sink_params_.channels(),
+                                             sink_params_.frames_per_buffer());
+  }
+
+  virtual ~WebRtcAudioConverter() {
+    DCHECK(thread_checker_.CalledOnValidThread());
+    audio_converter_.RemoveInput(this);
+  }
+
+  void Push(media::AudioBus* audio_source) {
+    // Called on the realtime audio thread, which must be the same thread as
+    // calling Convert().

[DaleCurtis, 2013/11/07 20:44:08]

Isn't this called on the capture thread while Convert() is called on the render
thread?  Also, Please use consistent naming instead of "realtime audio thread".

If Push() and Convert() must be called on the same thread, consider another
ThreadChecker which is bound on Push() or Convert().

[no longer working on chromium, 2013/11/08 13:01:15]

There are two Converters used by the processor: capture_converter_ and
render_converter_.
capture_converter_ is used to convert the capture data, and render_converter_ is
used to convert the render data.
The Push() and Convert() methods of capture_converter_ are called on the capture
audio thread, while render_converter_ is called on the render audio thread.
I updated the comment and hopefully make it more clear.
Done with adding another threadchecker, and I have to add a flag to avoid
unnecessary DetachFromThread() call.

+    fifo_->Push(audio_source);
+  }
+
+  bool Convert(webrtc::AudioFrame* out) {
+    // Called on realtime audio thread.
+    // Return false if there is no 10ms data in the FIFO.
+    if (fifo_->frames() < (source_params_.sample_rate() / 100))
+      return false;
+
+    // Convert 10ms data to the output format, this will trigger ProvideInput().
+    audio_converter_.Convert(audio_wrapper_.get());
+
+    // TODO(xians): Figure out a better way to handle the interleaved and
+    // deinterleaved format switching.
+    audio_wrapper_->ToInterleaved(audio_wrapper_->frames(),
+                                  sink_params_.bits_per_sample() / 8,
+                                  out->data_);
+
+    out->samples_per_channel_ = sink_params_.frames_per_buffer();
+    out->sample_rate_hz_ = sink_params_.sample_rate();
+    out->speech_type_ = webrtc::AudioFrame::kNormalSpeech;
+    out->vad_activity_ = webrtc::AudioFrame::kVadUnknown;
+    out->num_channels_ = sink_params_.channels();
+
+    return true;
+  }
+
+  const media::AudioParameters& source_parameters() const {
+    return source_params_;
+  }
+  const media::AudioParameters& sink_parameters() const {
+    return sink_params_;
+  }
+
+ private:
+  // AudioConverter::InputCallback implementation.
+  virtual double ProvideInput(media::AudioBus* audio_bus,
+                              base::TimeDelta buffer_delay) {
+    // Called on realtime audio thread.

[DaleCurtis, 2013/11/07 20:44:08]

Ditto.

[no longer working on chromium, 2013/11/08 13:01:15]

The same.

+    // TODO(xians): Figure out why the first Convert() triggers ProvideInput
+    // two times.
+    if (fifo_->frames() < audio_bus->frames())
+      return 0;
+
+    fifo_->Consume(audio_bus, 0, audio_bus->frames());
+    return 1.0;
+  }
+
+  base::ThreadChecker thread_checker_;
+  media::AudioParameters source_params_;
+  media::AudioParameters sink_params_;
+
+  // TODO(xians): consider using SincResampler to save some memcpy.
+  // Handles mixing and resampling between input and output parameters.
+  media::AudioConverter audio_converter_;
+  scoped_ptr<media::AudioBus> audio_wrapper_;
+  scoped_ptr<media::AudioFifo> fifo_;
+};
+
+WebRtcAudioProcessor::WebRtcAudioProcessor(
+    const webrtc::MediaConstraintsInterface* constraints)
+    : render_delay_ms_(0) {
+  InitializeAudioProcessingModule(constraints);
+}
+
+WebRtcAudioProcessor::~WebRtcAudioProcessor() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  StopAudioProcessing();
+}
+
+void WebRtcAudioProcessor::SetCaptureFormat(
+    const media::AudioParameters& source_params) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(source_params.IsValid());
+
+  // Create and initialize audio converter for the source data.
+  // When the webrtc AudioProcessing is enabled, the sink format of the
+  // converter will be the same as the post-processed data format, which is
+  // 32k mono for desktops and 16k mono for Android. When the AudioProcessing
+  // is disabled, the sink format will be the same as the source format.
+  const int sink_sample_rate = audio_processing_ ?
+      kAudioProcessingSampleRate : source_params.sample_rate();
+  const media::ChannelLayout sink_channel_layout = audio_processing_ ?
+      media::CHANNEL_LAYOUT_MONO : source_params.channel_layout();
+
+  // WebRtc is using 10ms data as its native packet size.
+  media::AudioParameters sink_params(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY, sink_channel_layout,
+      sink_sample_rate, 16, sink_sample_rate / 100);
+  capture_converter_.reset(
+      new WebRtcAudioConverter(source_params, sink_params));
+}
+
+void WebRtcAudioProcessor::PushCaptureData(media::AudioBus* audio_source) {
+  capture_converter_->Push(audio_source);
+}
+
+bool WebRtcAudioProcessor::ProcessAndConsumeData(
+    int capture_audio_delay_ms, int volume, bool key_pressed,
+    int16** out) {

[DaleCurtis, 2013/11/07 20:44:08]

Instead of int16**, should this just be a WebRtcAudioFrame?  Out is now pointing
at an internally owned construct.  it'd be better if whoever calls
ProcessAndConsumeData owned the data structure.

Alternatively you can copy the data out at this point.

[no longer working on chromium, 2013/11/08 13:01:15]

out is supposed to be used by all clients of the track, like webaudio, local
renderer, webrtc peer connection, so it'd better be generic type.
WebRtcAudioFrame is not a good candidate.
I would like to avoid the memcpy, if this does not look good to you, I can make
a accessor in WebRtcAudioProcessor like :
int16* output_data() const { capture_frame_.data_; }
And the client of WebRtcAudioProcessor can get the output data via the accesor.
wdyt?

+  TRACE_EVENT0("audio",
+               "WebRtcAudioProcessor::ProcessAndConsumeData");
+
+  if (!capture_converter_->Convert(&capture_frame_))
+    return false;
+
+  ProcessData(&capture_frame_, capture_audio_delay_ms, volume, key_pressed);
+  *out = capture_frame_.data_;
+
+  return true;
+}
+
+const media::AudioParameters& WebRtcAudioProcessor::OutputFormat() const {
+  return capture_converter_->sink_parameters();
+}
+
+void WebRtcAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame,

[DaleCurtis, 2013/11/07 20:44:08]

As with above, maybe having a
DCHECK(capture_thread_checker_.CalledOnValidThread());

and
DCHECK(render_thread_checker_.CalledOnValidThread());

might improve clarity of this code a lot.

[no longer working on chromium, 2013/11/08 13:01:15]

Done.

+                                       int capture_audio_delay_ms,
+                                       int volume,
+                                       bool key_pressed) {
+  if (!audio_processing_)
+    return;
+
+  TRACE_EVENT0("audio", "WebRtcAudioProcessor::Process10MsData");
+  DCHECK_EQ(audio_processing_->sample_rate_hz(),
+            capture_converter_->sink_parameters().sample_rate());
+  DCHECK_EQ(audio_processing_->num_input_channels(),
+            capture_converter_->sink_parameters().channels());
+  DCHECK_EQ(audio_processing_->num_output_channels(),
+            capture_converter_->sink_parameters().channels());
+
+  int total_delay_ms = 0;
+  {
+    base::AutoLock auto_lock(lock_);
+    total_delay_ms = capture_audio_delay_ms + render_delay_ms_;
+  }
+
+  audio_processing_->set_stream_delay_ms(total_delay_ms);
+  webrtc::GainControl* agc = audio_processing_->gain_control();
+  if (agc->set_stream_analog_level(volume))
+    NOTREACHED();
+  int err = audio_processing_->ProcessStream(audio_frame);
+  DCHECK(!err) << "ProcessStream() error: " << err;
+
+  // TODO(xians): Add support for AGC, typing detectin, audio level calculation,
+  // stereo swapping.
+}
+
+void WebRtcAudioProcessor::PushRenderData(
+    const int16* render_audio, int sample_rate, int number_of_channels,
+    int number_of_frames, int render_delay_ms) {
+  // Return immediately if the echo cancellation is off.
+  if (!audio_processing_ ||
+      !audio_processing_->echo_cancellation()->is_enabled())
+    return;
+
+  TRACE_EVENT0("audio",
+               "WebRtcAudioProcessor::FeedRenderDataToAudioProcessing");
+  {
+    base::AutoLock auto_lock(lock_);
+    render_delay_ms_ = render_delay_ms;
+  }
+
+  InitializeRenderConverterIfNeeded(sample_rate, number_of_channels,
+                                    number_of_frames);
+
+  // TODO(xians): Avoid this extra interleave/deinterleave.
+  render_data_bus_->FromInterleaved(render_audio,
+                                    render_data_bus_->frames(),
+                                    sizeof(render_audio[0]));
+  render_converter_->Push(render_data_bus_.get());
+  while (render_converter_->Convert(&render_frame_)) {
+    audio_processing_->AnalyzeReverseStream(&render_frame_);
+  }
+}
+
+void WebRtcAudioProcessor::InitializeAudioProcessingModule(
+    const webrtc::MediaConstraintsInterface* constraints) {
+  if (!CommandLine::ForCurrentProcess()->HasSwitch(
+          switches::kEnableAudioTrackProcessing)) {
+    return;
+  }
+
+  if (!constraints)
+    return;
+
+  const bool enable_aec = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kEchoCancellation);
+  const bool enable_ns = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kNoiseSuppression);
+  const bool enable_high_pass_filter = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kHighpassFilter);
+  const bool start_aec_dump = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kInternalAecDump);
+#if defined(IOS) || defined(ANDROID)
+  const bool enable_experimental_aec = false;
+  const bool enable_typing_detection = false;
+#else
+  const bool enable_experimental_aec = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kExperimentalEchoCancellation);
+  const bool enable_typing_detection = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kTypingNoiseDetection);
+#endif
+
+  // Reset the audio processing to NULL if no audio processing component is
+  // enabled.
+  if (!enable_aec && !enable_experimental_aec && !enable_ns &&
+      !enable_high_pass_filter && !enable_typing_detection) {
+    return;
+  }
+
+  // Create and configure the audio processing if it does not exist.
+  if (!audio_processing_)
+    audio_processing_.reset(webrtc::AudioProcessing::Create(0));
+
+  // Enable the audio processing components.
+  if (enable_aec) {
+    EnableEchoCancellation(audio_processing_.get());
+
+    if (enable_experimental_aec)
+      EnableExperimentalEchoCancellation(audio_processing_.get());
+  }
+
+  if (enable_ns)
+    EnableNoiseSuppression(audio_processing_.get());
+
+  if (enable_high_pass_filter)
+    EnableHighPassFilter(audio_processing_.get());
+
+  if (enable_typing_detection)
+    EnableTypingDetection(audio_processing_.get());
+
+  if (enable_aec && start_aec_dump)
+    StartAecDump(audio_processing_.get());
+
+  // Configure the audio format the audio processing is running on. This
+  // has to be done after all the needed components are enabled.
+  if (audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate))
+    NOTREACHED();
+  if (audio_processing_->set_num_channels(kAudioProcessingNumberOfChannel,
+                                          kAudioProcessingNumberOfChannel))
+    NOTREACHED();
+}
+
+void WebRtcAudioProcessor::InitializeRenderConverterIfNeeded(
+    int sample_rate, int number_of_channels, int frames_per_buffer) {
+  // TODO, figure out if we need to handle the buffer size change.
+  if (render_converter_.get() &&
+      render_converter_->source_parameters().sample_rate() == sample_rate &&
+      render_converter_->source_parameters().channels() == number_of_channels) {
+    // Do nothing if the |render_converter_| has been setup properly.
+    return;
+  }
+
+  media::AudioParameters source_params(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+      media::GuessChannelLayout(number_of_channels), sample_rate, 16,
+      frames_per_buffer);
+  media::AudioParameters sink_params(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+      media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16,
+      kAudioProcessingSampleRate / 100);
+  render_converter_.reset(new WebRtcAudioConverter(source_params, sink_params));
+  render_data_bus_ = media::AudioBus::Create(number_of_channels,
+                                             frames_per_buffer);
+}
+
+void WebRtcAudioProcessor::StopAudioProcessing() {
+  if (!audio_processing_.get())
+    return;
+
+  // It is safe to stop the AEC dump even it is not started.
+  StopAecDump(audio_processing_.get());
+
+  audio_processing_.reset();
+}
+
+}  // namespace content