Added an "enable-audio-processor" flag and WebRtcAudioProcessor class

content/renderer/media/webrtc_audio_processor.cc

+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/webrtc_audio_processor.h"
+
+#include "base/command_line.h"
+#include "base/debug/trace_event.h"
+#include "content/public/common/content_switches.h"
+#include "content/renderer/media/webrtc_audio_processor_options.h"
+#include "media/audio/audio_parameters.h"
+#include "media/base/audio_converter.h"
+#include "media/base/audio_fifo.h"
+#include "media/base/channel_layout.h"
+
+namespace content {
+
+namespace {
+
+using webrtc::AudioProcessing;
+using webrtc::MediaConstraintsInterface;
+
+#if defined(ANDROID)
+const int kAudioProcessingSampleRate = 16000;
+#else
+const int kAudioProcessingSampleRate = 32000;
+#endif
+const int kAudioProcessingNumberOfChannel = 1;
+
+const int kMaxNumberOfBuffersInFifo = 2;
+
+}  // namespace
+
+class WebRtcAudioProcessor::WebRtcAudioConverter
+    : public media::AudioConverter::InputCallback {
+ public:
+  WebRtcAudioConverter(const media::AudioParameters& source_params,
+                       const media::AudioParameters& sink_params)
+     : source_params_(source_params),
+       sink_params_(sink_params),
+       audio_converter_(source_params, sink_params_, false) {
+    worker_thread_checker_.DetachFromThread();
+
+    audio_converter_.AddInput(this);
+    // Create and initialize audio fifo and audio bus wrapper.
+    // The size of the FIFO should be at least twice of the source buffer size
+    // or twice of the sink buffer size.
+    int buffer_size = std::max(
+        kMaxNumberOfBuffersInFifo * source_params_.frames_per_buffer(),
+        kMaxNumberOfBuffersInFifo * sink_params_.frames_per_buffer());
+    fifo_.reset(new media::AudioFifo(source_params_.channels(), buffer_size));
+    // TODO(xians): Use CreateWrapper to save one memcpy.
+    audio_wrapper_ = media::AudioBus::Create(sink_params_.channels(),
+                                             sink_params_.frames_per_buffer());
+  }
+
+  virtual ~WebRtcAudioConverter() {
+    DCHECK(create_thread_checker_.CalledOnValidThread());
+    audio_converter_.RemoveInput(this);
+  }
+
+  void Push(media::AudioBus* audio_source) {
+    // Called on the audio thread, which is the capture audio thread for
+    // |WebRtcAudioProcessor::capture_converter_|, and render audio thread for
+    // |WebRtcAudioProcessor::render_converter_|.
+    // And it must be the same thread as calling Convert().
+    worker_thread_checker_.CalledOnValidThread();
+    fifo_->Push(audio_source);
+  }
+
+  bool Convert(webrtc::AudioFrame* out) {
+    // Called on the audio thread, which is the capture audio thread for
+    // |WebRtcAudioProcessor::capture_converter_|, and render audio thread for
+    // |WebRtcAudioProcessor::render_converter_|.
+    // Return false if there is no 10ms data in the FIFO.
+    worker_thread_checker_.CalledOnValidThread();
+    if (fifo_->frames() < (source_params_.sample_rate() / 100))
+      return false;
+
+    // Convert 10ms data to the output format, this will trigger ProvideInput().
+    audio_converter_.Convert(audio_wrapper_.get());
+
+    // TODO(xians): Figure out a better way to handle the interleaved and
+    // deinterleaved format switching.
+    audio_wrapper_->ToInterleaved(audio_wrapper_->frames(),
+                                  sink_params_.bits_per_sample() / 8,
+                                  out->data_);
+
+    out->samples_per_channel_ = sink_params_.frames_per_buffer();
+    out->sample_rate_hz_ = sink_params_.sample_rate();
+    out->speech_type_ = webrtc::AudioFrame::kNormalSpeech;
+    out->vad_activity_ = webrtc::AudioFrame::kVadUnknown;
+    out->num_channels_ = sink_params_.channels();
+
+    return true;
+  }
+
+  const media::AudioParameters& source_parameters() const {
+    return source_params_;
+  }
+  const media::AudioParameters& sink_parameters() const {
+    return sink_params_;
+  }
+
+ private:
+  // AudioConverter::InputCallback implementation.
+  virtual double ProvideInput(media::AudioBus* audio_bus,
+                              base::TimeDelta buffer_delay) {
+    // Called on realtime audio thread.
+    // TODO(xians): Figure out why the first Convert() triggers ProvideInput
+    // two times.
+    if (fifo_->frames() < audio_bus->frames())
+      return 0;
+
+    fifo_->Consume(audio_bus, 0, audio_bus->frames());
+    return 1.0;
+  }
+
+  base::ThreadChecker create_thread_checker_;
+  base::ThreadChecker worker_thread_checker_;
+  media::AudioParameters source_params_;
+  media::AudioParameters sink_params_;
+
+  // TODO(xians): consider using SincResampler to save some memcpy.
+  // Handles mixing and resampling between input and output parameters.
+  media::AudioConverter audio_converter_;
+  scoped_ptr<media::AudioBus> audio_wrapper_;
+  scoped_ptr<media::AudioFifo> fifo_;
+};
+
+WebRtcAudioProcessor::WebRtcAudioProcessor(
+    const webrtc::MediaConstraintsInterface* constraints) {
+  capture_thread_checker_.DetachFromThread();
+  render_thread_checker_.DetachFromThread();
+  InitializeAudioProcessingModule(constraints);
+}
+
+WebRtcAudioProcessor::~WebRtcAudioProcessor() {
+  DCHECK(main_thread_checker_.CalledOnValidThread());
+  StopAudioProcessing();
+}
+
+void WebRtcAudioProcessor::SetCaptureFormat(

[Henrik Grunell, 2013/11/12 12:46:37]

So this has to be called before the functions below. Or |capture_converter_|
isn't valid. Is this commented in the header file?

[no longer working on chromium, 2013/11/12 13:31:56]

Done with adding a comment in the header.

+    const media::AudioParameters& source_params) {
+  DCHECK(main_thread_checker_.CalledOnValidThread());
+  DCHECK(source_params.IsValid());
+
+  // Create and initialize audio converter for the source data.
+  // When the webrtc AudioProcessing is enabled, the sink format of the
+  // converter will be the same as the post-processed data format, which is
+  // 32k mono for desktops and 16k mono for Android. When the AudioProcessing
+  // is disabled, the sink format will be the same as the source format.
+  const int sink_sample_rate = audio_processing_ ?
+      kAudioProcessingSampleRate : source_params.sample_rate();
+  const media::ChannelLayout sink_channel_layout = audio_processing_ ?
+      media::CHANNEL_LAYOUT_MONO : source_params.channel_layout();
+
+  // WebRtc is using 10ms data as its native packet size.
+  media::AudioParameters sink_params(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY, sink_channel_layout,
+      sink_sample_rate, 16, sink_sample_rate / 100);
+  capture_converter_.reset(
+      new WebRtcAudioConverter(source_params, sink_params));
+}
+
+void WebRtcAudioProcessor::PushCaptureData(media::AudioBus* audio_source) {
+  capture_thread_checker_.CalledOnValidThread();
+  capture_converter_->Push(audio_source);
+}
+
+bool WebRtcAudioProcessor::ProcessAndConsumeData(
+    base::TimeDelta capture_delay, int volume, bool key_pressed,
+    int16** out) {
+  capture_thread_checker_.CalledOnValidThread();
+  TRACE_EVENT0("audio",
+               "WebRtcAudioProcessor::ProcessAndConsumeData");
+
+  if (!capture_converter_->Convert(&capture_frame_))
+    return false;
+
+  ProcessData(&capture_frame_, capture_delay, volume, key_pressed);
+  *out = capture_frame_.data_;
+
+  return true;
+}
+
+const media::AudioParameters& WebRtcAudioProcessor::OutputFormat() const {
+  return capture_converter_->sink_parameters();
+}
+
+void WebRtcAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame,
+                                       base::TimeDelta capture_delay,
+                                       int volume,
+                                       bool key_pressed) {
+  capture_thread_checker_.CalledOnValidThread();
+  if (!audio_processing_)
+    return;
+
+  TRACE_EVENT0("audio", "WebRtcAudioProcessor::Process10MsData");
+  DCHECK_EQ(audio_processing_->sample_rate_hz(),
+            capture_converter_->sink_parameters().sample_rate());
+  DCHECK_EQ(audio_processing_->num_input_channels(),
+            capture_converter_->sink_parameters().channels());
+  DCHECK_EQ(audio_processing_->num_output_channels(),
+            capture_converter_->sink_parameters().channels());
+
+  base::subtle::Atomic32 render_delay_ms =
+      base::subtle::NoBarrier_Load(&render_delay_ms_);
+  int64 capture_delay_ms = capture_delay.InMilliseconds();
+  CHECK_LT(capture_delay_ms, INT_MAX);
+  int total_delay_ms =  capture_delay_ms + render_delay_ms;
+  if (total_delay_ms > 1000) {
+    LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms

[Henrik Grunell, 2013/11/12 12:46:37]

Should we really log a warning here? What if it gets stuck in a large delay,
lots of warnings.

[no longer working on chromium, 2013/11/12 13:31:56]

Yes, when the delay is unusually large, we would like to get the warning log.

+                 << "ms; render delay: " << render_delay_ms << "ms";
+  }
+
+  audio_processing_->set_stream_delay_ms(total_delay_ms);
+  webrtc::GainControl* agc = audio_processing_->gain_control();
+  if (agc->set_stream_analog_level(volume))
+    NOTREACHED();
+  int err = audio_processing_->ProcessStream(audio_frame);
+  DCHECK(!err) << "ProcessStream() error: " << err;
+
+  // TODO(xians): Add support for AGC, typing detectin, audio level calculation,

[Henrik Grunell, 2013/11/12 12:46:37]

Nit: detection

[no longer working on chromium, 2013/11/12 13:31:56]

Done.

+  // stereo swapping.
+}
+
+void WebRtcAudioProcessor::PushRenderData(
+    const int16* render_audio, int sample_rate, int number_of_channels,
+    int number_of_frames, base::TimeDelta render_delay) {
+  render_thread_checker_.CalledOnValidThread();
+
+  // Return immediately if the echo cancellation is off.
+  if (!audio_processing_ ||
+      !audio_processing_->echo_cancellation()->is_enabled())
+    return;
+
+  TRACE_EVENT0("audio",
+               "WebRtcAudioProcessor::FeedRenderDataToAudioProcessing");
+  int64 new_render_delay_ms = render_delay.InMilliseconds();
+  CHECK_LT(new_render_delay_ms, INT_MAX);
+  base::subtle::Atomic32 render_delay_ms = new_render_delay_ms;
+  base::subtle::NoBarrier_Store(&render_delay_ms_, render_delay_ms);
+
+  InitializeRenderConverterIfNeeded(sample_rate, number_of_channels,
+                                    number_of_frames);
+
+  // TODO(xians): Avoid this extra interleave/deinterleave.
+  render_data_bus_->FromInterleaved(render_audio,
+                                    render_data_bus_->frames(),
+                                    sizeof(render_audio[0]));
+  render_converter_->Push(render_data_bus_.get());
+  while (render_converter_->Convert(&render_frame_)) {
+    audio_processing_->AnalyzeReverseStream(&render_frame_);
+  }
+}
+
+void WebRtcAudioProcessor::InitializeAudioProcessingModule(
+    const webrtc::MediaConstraintsInterface* constraints) {
+  if (!CommandLine::ForCurrentProcess()->HasSwitch(
+          switches::kEnableAudioTrackProcessing)) {
+    return;
+  }
+
+  if (!constraints)
+    return;
+
+  const bool enable_aec = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kEchoCancellation);
+  const bool enable_ns = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kNoiseSuppression);
+  const bool enable_high_pass_filter = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kHighpassFilter);
+  const bool start_aec_dump = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kInternalAecDump);
+#if defined(IOS) || defined(ANDROID)
+  const bool enable_experimental_aec = false;
+  const bool enable_typing_detection = false;
+#else
+  const bool enable_experimental_aec = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kExperimentalEchoCancellation);
+  const bool enable_typing_detection = GetPropertyFromConstraints(
+      constraints, MediaConstraintsInterface::kTypingNoiseDetection);
+#endif
+
+  // Reset the audio processing to NULL if no audio processing component is

[Henrik Grunell, 2013/11/12 12:46:37]

I don't see a reset here. Old comment?

[no longer working on chromium, 2013/11/12 13:31:56]

Fixed the comment.

+  // enabled.
+  if (!enable_aec && !enable_experimental_aec && !enable_ns &&
+      !enable_high_pass_filter && !enable_typing_detection) {
+    return;
+  }
+
+  // Create and configure the audio processing if it does not exist.
+  if (!audio_processing_)
+    audio_processing_.reset(webrtc::AudioProcessing::Create(0));
+
+  // Enable the audio processing components.
+  if (enable_aec) {
+    EnableEchoCancellation(audio_processing_.get());
+

[Henrik Grunell, 2013/11/12 12:46:37]

Nit: I think you can skip the blank line.

[no longer working on chromium, 2013/11/12 13:31:56]

Done.

+    if (enable_experimental_aec)
+      EnableExperimentalEchoCancellation(audio_processing_.get());
+  }
+
+  if (enable_ns)
+    EnableNoiseSuppression(audio_processing_.get());
+
+  if (enable_high_pass_filter)
+    EnableHighPassFilter(audio_processing_.get());
+
+  if (enable_typing_detection)
+    EnableTypingDetection(audio_processing_.get());
+
+  if (enable_aec && start_aec_dump)
+    StartAecDump(audio_processing_.get());
+
+  // Configure the audio format the audio processing is running on. This
+  // has to be done after all the needed components are enabled.
+  if (audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate))
+    NOTREACHED();
+  if (audio_processing_->set_num_channels(kAudioProcessingNumberOfChannel,
+                                          kAudioProcessingNumberOfChannel))
+    NOTREACHED();
+}
+
+void WebRtcAudioProcessor::InitializeRenderConverterIfNeeded(
+    int sample_rate, int number_of_channels, int frames_per_buffer) {
+  // TODO(xians): Figure out if we need to handle the buffer size change.
+  if (render_converter_.get() &&
+      render_converter_->source_parameters().sample_rate() == sample_rate &&
+      render_converter_->source_parameters().channels() == number_of_channels) {
+    // Do nothing if the |render_converter_| has been setup properly.
+    return;
+  }
+
+  media::AudioParameters source_params(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+      media::GuessChannelLayout(number_of_channels), sample_rate, 16,
+      frames_per_buffer);
+  media::AudioParameters sink_params(
+      media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+      media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16,
+      kAudioProcessingSampleRate / 100);
+  render_converter_.reset(new WebRtcAudioConverter(source_params, sink_params));
+  render_data_bus_ = media::AudioBus::Create(number_of_channels,
+                                             frames_per_buffer);
+}
+
+void WebRtcAudioProcessor::StopAudioProcessing() {
+  if (!audio_processing_.get())
+    return;
+
+  // It is safe to stop the AEC dump even it is not started.
+  StopAecDump(audio_processing_.get());
+
+  audio_processing_.reset();
+}
+
+}  // namespace content