Binding media stream audio track to speech recognition [browser]

content/browser/speech/speech_recognizer_impl.cc

Index: content/browser/speech/speech_recognizer_impl.cc
diff --git a/content/browser/speech/speech_recognizer_impl.cc b/content/browser/speech/speech_recognizer_impl.cc
index 9fe98be67eabe9247609fd88c87d57a3c013afb4..e47655c762c45fe00c4c35027b36942ac70ae5d3 100644
--- a/content/browser/speech/speech_recognizer_impl.cc
+++ b/content/browser/speech/speech_recognizer_impl.cc
@@ -6,11 +6,16 @@
 
 #include "base/basictypes.h"
 #include "base/bind.h"
+#include "base/memory/shared_memory.h"
+#include "base/threading/thread.h"
+#include "base/threading/thread_restrictions.h"
 #include "base/time/time.h"
 #include "content/browser/browser_main_loop.h"
 #include "content/browser/media/media_internals.h"
 #include "content/browser/speech/audio_buffer.h"
 #include "content/browser/speech/google_one_shot_remote_engine.h"
+#include "content/browser/speech/speech_recognition_audio_receiver.h"
+#include "content/public/browser/browser_message_filter.h"
 #include "content/public/browser/speech_recognition_event_listener.h"
 #include "media/base/audio_converter.h"
 #include "net/url_request/url_request_context_getter.h"
@@ -41,6 +46,7 @@ class SpeechRecognizerImpl::OnDataConverter
   // Converts input audio |data| bus into an AudioChunk where the input format
   // is given by |input_parameters_| and the output format by
   // |output_parameters_|.
+  // Skips conversion on |audio_converter_| when input and output formats match.
   scoped_refptr<AudioChunk> Convert(const AudioBus* data);
 
  private:
@@ -57,6 +63,7 @@ class SpeechRecognizerImpl::OnDataConverter
   const AudioParameters input_parameters_;
   const AudioParameters output_parameters_;
   bool waiting_for_input_;
+  bool need_conversion_;
   scoped_ptr<uint8[]> converted_data_;
 
   DISALLOW_COPY_AND_ASSIGN(OnDataConverter);
@@ -124,6 +131,7 @@ SpeechRecognizerImpl::OnDataConverter::OnDataConverter(
       waiting_for_input_(false),
       converted_data_(new uint8[output_parameters_.GetBytesPerBuffer()]) {
   audio_converter_.AddInput(this);
+  need_conversion_ = !input_parameters_.Equals(output_parameters_);
 }
 
 SpeechRecognizerImpl::OnDataConverter::~OnDataConverter() {
@@ -136,10 +144,15 @@ scoped_refptr<AudioChunk> SpeechRecognizerImpl::OnDataConverter::Convert(
     const AudioBus* data) {
   CHECK_EQ(data->frames(), input_parameters_.frames_per_buffer());
 
-  data->CopyTo(input_bus_.get());
-
-  waiting_for_input_ = true;
-  audio_converter_.Convert(output_bus_.get());
+  // Do conversion if needed - only when |input_parameters_| differ from
+  // |output_parameters_|.
+  if (need_conversion_) {
+    data->CopyTo(input_bus_.get());
+    waiting_for_input_ = true;
+    audio_converter_.Convert(output_bus_.get());
+  } else {
+    data->CopyTo(output_bus_.get());
+  }
 
   output_bus_->ToInterleaved(
       output_bus_->frames(), output_parameters_.bits_per_sample() / 8,
@@ -175,6 +188,8 @@ SpeechRecognizerImpl::SpeechRecognizerImpl(
     int session_id,
     bool continuous,
     bool provisional_results,
+    bool using_audio_receiver,
+    const base::ProcessHandle peer_process_handle,
     SpeechRecognitionEngine* engine)
     : SpeechRecognizer(listener, session_id),
       recognition_engine_(engine),
@@ -183,6 +198,8 @@ SpeechRecognizerImpl::SpeechRecognizerImpl(
           media::AudioLogFactory::AUDIO_INPUT_CONTROLLER)),
       is_dispatching_event_(false),
       provisional_results_(provisional_results),
+      using_audio_receiver_(using_audio_receiver),
+      peer_process_handle_(peer_process_handle),
       state_(STATE_IDLE) {
   DCHECK(recognition_engine_ != NULL);
   if (!continuous) {
@@ -214,7 +231,7 @@ SpeechRecognizerImpl::SpeechRecognizerImpl(
 // synchronous callbacks.
 
 void SpeechRecognizerImpl::StartRecognition(const std::string& device_id) {
-  DCHECK(!device_id.empty());
+  DCHECK(!device_id.empty() ^ using_audio_receiver_);
   device_id_ = device_id;
 
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
@@ -245,7 +262,8 @@ bool SpeechRecognizerImpl::IsCapturingAudio() const {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); // See IsActive().
   const bool is_capturing_audio = state_ >= STATE_STARTING &&
                                   state_ <= STATE_RECOGNIZING;
-  DCHECK((is_capturing_audio && (audio_controller_.get() != NULL)) ||
+  DCHECK((is_capturing_audio &&
+          (using_audio_receiver_ || (audio_controller_.get() != NULL))) ||
          (!is_capturing_audio && audio_controller_.get() == NULL));
   return is_capturing_audio;
 }
@@ -262,6 +280,7 @@ SpeechRecognizerImpl::~SpeechRecognizerImpl() {
     audio_controller_->Close(
         base::Bind(&KeepAudioControllerRefcountedForDtor, audio_controller_));
     audio_log_->OnClosed(0);
+    ResetAudioReceiver();
   }
 }
 
@@ -482,7 +501,7 @@ void SpeechRecognizerImpl::ProcessAudioPipeline(const AudioChunk& raw_audio) {
     endpointer_.ProcessAudio(raw_audio, &rms);
 
   if (route_to_vumeter) {
-    DCHECK(route_to_endpointer); // Depends on endpointer due to |rms|.
+    DCHECK(route_to_endpointer);  // Depends on endpointer due to |rms|.
     UpdateSignalAndNoiseLevels(rms, clip_detected);
   }
   if (route_to_sr_engine) {
@@ -529,6 +548,16 @@ SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
       AudioParameters::AUDIO_PCM_LOW_LATENCY, kChannelLayout, kAudioSampleRate,
       kNumBitsPerAudioSample, frames_per_buffer);
 
+  // Check if using audio receiver as source (audio data processed on renderer).
+  // If |StartRecordingUsingAudioReceiver()| fails, we abort the recognition.
+  if (using_audio_receiver_) {
+    return StartRecordingUsingAudioReceiver(output_parameters)
+               ? STATE_STARTING
+               : Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
+  }
+
+  // Get audio data from microphone on the browser.
+
   // Audio converter will receive audio based on these parameters as input.
   // On Windows we start by verifying that Core Audio is supported. If not,
   // the WaveIn API is used and we might as well avoid all audio conversations
@@ -585,6 +614,70 @@ SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
   return STATE_STARTING;
 }
 
+bool SpeechRecognizerImpl::StartRecordingUsingAudioReceiver(
+    const media::AudioParameters& params) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  audio_thread_.reset(new base::Thread("AudioReceiverThread"));

[tommi (sloooow) - chröme, 2014/10/16 21:20:23]

what if this is already set? (should we at least DCHECK that it's null first?)

[burnik, 2014/10/17 10:48:02]

Done. However:
That would indicate the FSM contract is broken, would fail earlier when checking
if audio is being captured.

+  if (!audio_thread_->Start()) {
+    DLOG(ERROR) << "Cannot start new audio thread";

[tommi (sloooow) - chröme, 2014/10/16 21:20:23]

free audio_thread_?

[burnik, 2014/10/17 10:48:02]

Done.

+    return false;
+  }
+
+  // Start receiving audio, incoming data will be posted to |OnAudioTrackData|.
+  base::SharedMemoryHandle foreign_memory_handle;
+  base::SyncSocket::TransitDescriptor foreign_socket_descriptor;
+  audio_receiver_ = SpeechRecognitionAudioReceiver::Create(
+      params,
+      peer_process_handle_,
+      this,  // Trigger OnData() and OnError() events here.
+      audio_thread_->task_runner().get(),
+      &foreign_memory_handle,
+      &foreign_socket_descriptor);
+
+  if (!audio_receiver_.get()) {
+    ResetAudioReceiver();
+    return false;
+  }
+
+  // Reusing the audio converter only to create chunks from the bus.
+  // Same input and output format indicate no additional conversion here.
+  audio_converter_.reset(new OnDataConverter(params, params));
+
+  // The endpointer needs to estimate the environment/background noise before
+  // starting to treat the audio as user input. We wait in the state
+  // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching
+  // to user input mode.
+  endpointer_.SetEnvironmentEstimationMode();
+
+  // Start listening on the receiver's audio thread.
+  audio_receiver_->Listen();
+
+  listener()->OnAudioReceiverReady(session_id(),
+                                   params,
+                                   foreign_memory_handle,
+                                   foreign_socket_descriptor);
+
+  return true;
+}
+
+void SpeechRecognizerImpl::ResetAudioReceiver() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  // We stop listening not to post any new tasks to the audio thread.
+  audio_receiver_->Stop();
+
+  // This synchronous operation is usually instantaneous since the receiver
+  // closes the socket on Stop() and no new tasks will be posted to the
+  // |audio_thread_|.
+  const bool previous_io_allowed = base::ThreadRestrictions::SetIOAllowed(true);

[tommi (sloooow) - chröme, 2014/10/16 21:20:23]

ScopedAllowIO

[burnik, 2014/10/17 10:48:02]

Unfortunately, ScopedAllowIO is deprecated and is part of the pre-submit checks
which block the CL. I realize this is usually discouraged, but here it's
commented and does not really block.

Side note:
Starting and stopping the thread on the BrowserThread::FILE would make this very
complicated (async callbacks calling async callbacks) and would probably require
a lock which would again lead us back to a synchronous operation. If the rest of
the class was implemented on the FILE thread, we would have no such issue
(although other problems would most probably emerge).

+  audio_thread_->Stop();
+  base::ThreadRestrictions::SetIOAllowed(previous_io_allowed);
+
+  audio_thread_.reset();
+  audio_receiver_.reset();
+}
+
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::StartRecognitionEngine(const FSMEventArgs& event_args) {
   // This is the first audio packet captured, so the recognition engine is
@@ -667,6 +760,7 @@ SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::Abort(
   if (IsCapturingAudio())
     CloseAudioControllerAsynchronously();
 
+
   DVLOG(1) << "SpeechRecognizerImpl canceling recognition. ";
 
   // The recognition engine is initialized only after STATE_STARTING.
@@ -770,6 +864,12 @@ SpeechRecognizerImpl::NotFeasible(const FSMEventArgs& event_args) {
 
 void SpeechRecognizerImpl::CloseAudioControllerAsynchronously() {
   DCHECK(IsCapturingAudio());
+
+  if (using_audio_receiver_) {
+    ResetAudioReceiver();
+    return;
+  }
+
   DVLOG(1) << "SpeechRecognizerImpl closing audio controller.";
   // Issues a Close on the audio controller, passing an empty callback. The only
   // purpose of such callback is to keep the audio controller refcounted until