Adding typing detection to the APM in chrome.

content/renderer/media/media_stream_audio_processor.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/media_stream_audio_processor.h"
 
 #include "base/command_line.h"
 #include "base/debug/trace_event.h"
 #include "content/public/common/content_switches.h"
 #include "content/renderer/media/media_stream_audio_processor_options.h"
 #include "content/renderer/media/rtc_media_constraints.h"
 #include "media/audio/audio_parameters.h"
 #include "media/base/audio_converter.h"
 #include "media/base/audio_fifo.h"
 #include "media/base/channel_layout.h"
 #include "third_party/WebKit/public/platform/WebMediaConstraints.h"
 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface.h"
+#include "third_party/webrtc/modules/audio_processing/typing_detection.h"
 
 namespace content {
 
 namespace {
 
 using webrtc::AudioProcessing;
 using webrtc::MediaConstraintsInterface;
 
 #if defined(OS_ANDROID)
 const int kAudioProcessingSampleRate = 16000;
@@ skipping 108 matching lines @@
   media::AudioConverter audio_converter_;
   scoped_ptr<media::AudioBus> audio_wrapper_;
   scoped_ptr<media::AudioFifo> fifo_;
 };
 
 MediaStreamAudioProcessor::MediaStreamAudioProcessor(
     const media::AudioParameters& source_params,
     const blink::WebMediaConstraints& constraints,
     int effects)
     : render_delay_ms_(0),
-      audio_mirroring_(false) {
+      audio_mirroring_(false),
+      typing_detected_(false) {
   capture_thread_checker_.DetachFromThread();
   render_thread_checker_.DetachFromThread();
   InitializeAudioProcessingModule(constraints, effects);
   InitializeCaptureConverter(source_params);
 }
 
 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() {
   DCHECK(main_thread_checker_.CalledOnValidThread());
   StopAudioProcessing();
 }
@@ skipping 100 matching lines @@
   const bool enable_ns = GetPropertyFromConstraints(
       &native_constraints, MediaConstraintsInterface::kNoiseSuppression);
   const bool enable_high_pass_filter = GetPropertyFromConstraints(
       &native_constraints, MediaConstraintsInterface::kHighpassFilter);
 
   audio_mirroring_ = GetPropertyFromConstraints(
       &native_constraints, webrtc::MediaConstraintsInterface::kAudioMirroring);
 
   // Return immediately if no audio processing component is enabled.
   if (!enable_aec && !enable_experimental_aec && !enable_ns &&
-      !enable_high_pass_filter && !enable_typing_detection && !enable_agc) {
+      !enable_high_pass_filter && !enable_typing_detection && !enable_agc &&
+      !audio_mirroring_) {
     return;
   }
 
   // Create and configure the webrtc::AudioProcessing.
   audio_processing_.reset(webrtc::AudioProcessing::Create(0));
 
   // Enable the audio processing components.
   if (enable_aec) {
     EnableEchoCancellation(audio_processing_.get());
     if (enable_experimental_aec)
       EnableExperimentalEchoCancellation(audio_processing_.get());
   }
 
   if (enable_ns)
     EnableNoiseSuppression(audio_processing_.get());
 
   if (enable_high_pass_filter)
     EnableHighPassFilter(audio_processing_.get());
 
-  if (enable_typing_detection)
+  if (enable_typing_detection) {
     EnableTypingDetection(audio_processing_.get());

[Henrik Grunell, 2014/02/19 06:54:42]

The semantics/naming is a bit unclear. EnableTypingDetection() doesn't really
_enable_ typing detection, it sets up |audio_processing_| so that typing
detection can get the information it needs. Which is enable VAD. What do you
think about simply renaming that function to EnableVoiceActivityDetection()?

Alternatively, have EnableTypingDetection() take both |audio_processing_| and
|typing_detector_| as parameters and do typing_detector_->SetParameters() there.

[no longer working on chromium, 2014/02/20 12:44:13]

The naming of EnableTypingDetection is corresponding to kTypingNoiseDetection
constraint, and it does not need to know the details of what APM is doing.

So I think we should keep the existing name, and move typing_detector_ inside
EnableTypingDetection.

+    // TODO(xians): Remove this |typing_detector_| after the typing suppression
+    // is enabled by default.
+    typing_detector_.reset(new webrtc::TypingDetection());
+
+    // Configure the update period to 100ms (10 * 10ms) in the typing detector.

[Henrik Grunell, 2014/02/19 06:54:42]

Nit: space between number and "ms".

[no longer working on chromium, 2014/02/20 12:44:13]

Discussed offline, in Chrome, it is very common to take the number and ms
together.

+    typing_detector_->SetParameters(0, 0, 0, 0, 0, 10);
+  }
 
   if (enable_agc)
     EnableAutomaticGainControl(audio_processing_.get());
 
   // Configure the audio format the audio processing is running on. This
   // has to be done after all the needed components are enabled.
   CHECK_EQ(audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate),
            0);
   CHECK_EQ(audio_processing_->set_num_channels(kAudioProcessingNumberOfChannel,
                                                kAudioProcessingNumberOfChannel),
@@ skipping 92 matching lines @@
   DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err;
   err = audio_processing_->ProcessStream(audio_frame);
   DCHECK_EQ(err, 0) << "ProcessStream() error: " << err;
 
   // TODO(xians): Add support for typing detection, audio level calculation.
 
   if (audio_mirroring_ && audio_frame->num_channels_ == 2) {
     // TODO(xians): Swap the stereo channels after switching to media::AudioBus.
   }
 
+  if (typing_detector_ &&
+      audio_frame->vad_activity_ != webrtc::AudioFrame::kVadUnknown) {
+    bool vad_active =
+        (audio_frame->vad_activity_ == webrtc::AudioFrame::kVadActive);
+    // TODO(xians): Pass this |typing_detected_| to peer connection.
+    typing_detected_ = typing_detector_->Process(key_pressed, vad_active);
+  }
+
   // Return 0 if the volume has not been changed, otherwise return the new
   // volume.
   return (agc->stream_analog_level() == volume) ?
       0 : agc->stream_analog_level();
 }
 
 void MediaStreamAudioProcessor::StopAudioProcessing() {
   if (!audio_processing_.get())
     return;
 
   audio_processing_.reset();
 }
 
 }  // namespace content