Switch Speech Recognition to asynchronous callback-based AudioManager interactions.

content/browser/speech/speech_recognizer_impl.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/speech/speech_recognizer_impl.h"
 
 #include <stdint.h>
 
 #include <algorithm>
 
 #include "base/bind.h"
 #include "base/macros.h"
 #include "base/time/time.h"
 #include "build/build_config.h"
 #include "content/browser/browser_main_loop.h"
 #include "content/browser/media/media_internals.h"
 #include "content/browser/speech/audio_buffer.h"
 #include "content/public/browser/speech_recognition_event_listener.h"
 #include "media/audio/audio_file_writer.h"
+#include "media/audio/audio_manager.h"
+#include "media/audio/audio_system.h"
 #include "media/base/audio_converter.h"
 
 #if defined(OS_WIN)
 #include "media/audio/win/core_audio_util_win.h"
 #endif
 
 using media::AudioBus;
 using media::AudioConverter;
 using media::AudioInputController;
 using media::AudioManager;
@@ skipping 75 matching lines @@
 }
 
 }  // namespace
 
 const int SpeechRecognizerImpl::kAudioSampleRate = 16000;
 const ChannelLayout SpeechRecognizerImpl::kChannelLayout =
     media::CHANNEL_LAYOUT_MONO;
 const int SpeechRecognizerImpl::kNumBitsPerAudioSample = 16;
 const int SpeechRecognizerImpl::kNoSpeechTimeoutMs = 8000;
 const int SpeechRecognizerImpl::kEndpointerEstimationTimeMs = 300;
-media::AudioManager* SpeechRecognizerImpl::audio_manager_for_tests_ = NULL;
+media::AudioSystem* SpeechRecognizerImpl::audio_system_for_tests_ = nullptr;
 
 static_assert(SpeechRecognizerImpl::kNumBitsPerAudioSample % 8 == 0,
               "kNumBitsPerAudioSample must be a multiple of 8");
 
 // SpeechRecognizerImpl::OnDataConverter implementation
 
 SpeechRecognizerImpl::OnDataConverter::OnDataConverter(
     const AudioParameters& input_params,
     const AudioParameters& output_params)
     : audio_converter_(input_params, output_params, false),
@@ skipping 43 matching lines @@
   input_bus_->CopyTo(dest);
   // Indicate that the recorded audio has in fact been used by the converter.
   data_was_converted_ = true;
   return 1;
 }
 
 // SpeechRecognizerImpl implementation
 
 SpeechRecognizerImpl::SpeechRecognizerImpl(
     SpeechRecognitionEventListener* listener,
+    media::AudioSystem* audio_system,
     int session_id,
     bool continuous,
     bool provisional_results,
     SpeechRecognitionEngine* engine)
     : SpeechRecognizer(listener, session_id),
+      audio_system_(audio_system),
       recognition_engine_(engine),
       endpointer_(kAudioSampleRate),
       audio_log_(MediaInternals::GetInstance()->CreateAudioLog(
           media::AudioLogFactory::AUDIO_INPUT_CONTROLLER)),
       is_dispatching_event_(false),
       provisional_results_(provisional_results),
       end_of_utterance_(false),
-      state_(STATE_IDLE) {
-  DCHECK(recognition_engine_ != NULL);
+      state_(STATE_IDLE),
+      weak_ptr_factory_(this) {
+  DCHECK(recognition_engine_ != nullptr);
+  DCHECK(audio_system_ != nullptr);
   if (!continuous) {
     // In single shot (non-continous) recognition,
     // the session is automatically ended after:
     //  - 0.5 seconds of silence if time <  3 seconds
     //  - 1   seconds of silence if time >= 3 seconds
     endpointer_.set_speech_input_complete_silence_length(
         base::Time::kMicrosecondsPerSecond / 2);
     endpointer_.set_long_speech_input_complete_silence_length(
         base::Time::kMicrosecondsPerSecond);
     endpointer_.set_long_speech_length(3 * base::Time::kMicrosecondsPerSecond);
@@ skipping 13 matching lines @@
 // NOTE:all the external events and requests should be enqueued (PostTask), even
 // if they come from the same (IO) thread, in order to preserve the relationship
 // of causality between events and avoid interleaved event processing due to
 // synchronous callbacks.
 
 void SpeechRecognizerImpl::StartRecognition(const std::string& device_id) {
   DCHECK(!device_id.empty());
   device_id_ = device_id;
 
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
-                          base::Bind(&SpeechRecognizerImpl::DispatchEvent,
-                                     this, FSMEventArgs(EVENT_START)));
+                          base::Bind(&SpeechRecognizerImpl::DispatchEvent, this,
+                                     FSMEventArgs(EVENT_PREPARE)));
 }
 
 void SpeechRecognizerImpl::AbortRecognition() {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
                                      this, FSMEventArgs(EVENT_ABORT)));
 }
 
 void SpeechRecognizerImpl::StopAudioCapture() {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
@@ skipping 131 matching lines @@
 SpeechRecognizerImpl::ExecuteTransitionAndGetNextState(
     const FSMEventArgs& event_args) {
   const FSMEvent event = event_args.event;
   switch (state_) {
     case STATE_IDLE:
       switch (event) {
         // TODO(primiano): restore UNREACHABLE_CONDITION on EVENT_ABORT and
         // EVENT_STOP_CAPTURE below once speech input extensions are fixed.
         case EVENT_ABORT:
           return AbortSilently(event_args);
+        case EVENT_PREPARE:
+          return PrepareRecognition(event_args);
+        case EVENT_START:
+          return NotFeasible(event_args);
+        case EVENT_STOP_CAPTURE:
+          return AbortSilently(event_args);
+        case EVENT_AUDIO_DATA:     // Corner cases related to queued messages
+        case EVENT_ENGINE_RESULT:  // being lately dispatched.
+        case EVENT_ENGINE_ERROR:
+        case EVENT_AUDIO_ERROR:
+          return DoNothing(event_args);
+      }
+      break;
+    case STATE_PREPARING:
+      switch (event) {
+        case EVENT_ABORT:
+          return AbortSilently(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return StartRecording(event_args);
         case EVENT_STOP_CAPTURE:
           return AbortSilently(event_args);
         case EVENT_AUDIO_DATA:     // Corner cases related to queued messages
         case EVENT_ENGINE_RESULT:  // being lately dispatched.
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return DoNothing(event_args);
       }
       break;
     case STATE_STARTING:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return AbortSilently(event_args);
         case EVENT_AUDIO_DATA:
           return StartRecognitionEngine(event_args);
         case EVENT_ENGINE_RESULT:
           return NotFeasible(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_ESTIMATING_ENVIRONMENT:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return StopCaptureAndWaitForResult(event_args);
         case EVENT_AUDIO_DATA:
           return WaitEnvironmentEstimationCompletion(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessIntermediateResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_WAITING_FOR_SPEECH:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return StopCaptureAndWaitForResult(event_args);
         case EVENT_AUDIO_DATA:
           return DetectUserSpeechOrTimeout(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessIntermediateResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_RECOGNIZING:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return StopCaptureAndWaitForResult(event_args);
         case EVENT_AUDIO_DATA:
           return DetectEndOfSpeech(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessIntermediateResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_WAITING_FINAL_RESULT:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
         case EVENT_AUDIO_DATA:
           return DoNothing(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessFinalResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
@@ skipping 36 matching lines @@
   if (route_to_vumeter) {
     DCHECK(route_to_endpointer);  // Depends on endpointer due to |rms|.
     UpdateSignalAndNoiseLevels(rms, clip_detected);
   }
   if (route_to_sr_engine) {
     DCHECK(recognition_engine_.get() != NULL);
     recognition_engine_->TakeAudioChunk(raw_audio);
   }
 }
 
+void SpeechRecognizerImpl::OnDeviceInfo(const media::AudioParameters& params) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  device_params_ = params;
+  DVLOG(1) << "Device parameters: " << device_params_.AsHumanReadableString();
+  DispatchEvent(FSMEventArgs(EVENT_START));
+}
+
+SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::PrepareRecognition(
+    const FSMEventArgs&) {
+  DCHECK(state_ == STATE_IDLE);
+  DCHECK(recognition_engine_.get() != NULL);
+  DCHECK(!IsCapturingAudio());
+  GetAudioSystem()->GetInputStreamParameters(
+      device_id_, base::Bind(&SpeechRecognizerImpl::OnDeviceInfo,
+                             weak_ptr_factory_.GetWeakPtr()));
+
+  listener()->OnRecognitionStart(session_id());
+  return STATE_PREPARING;
+}
+
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
-  DCHECK(state_ == STATE_IDLE);
+  DCHECK(state_ == STATE_PREPARING);
   DCHECK(recognition_engine_.get() != NULL);
   DCHECK(!IsCapturingAudio());
-  const bool unit_test_is_active = (audio_manager_for_tests_ != NULL);
-  AudioManager* audio_manager = unit_test_is_active ?
-                                audio_manager_for_tests_ :
-                                AudioManager::Get();
-  DCHECK(audio_manager != NULL);
 
   DVLOG(1) << "SpeechRecognizerImpl starting audio capture.";
   num_samples_recorded_ = 0;
   audio_level_ = 0;
   end_of_utterance_ = false;
-  listener()->OnRecognitionStart(session_id());
 
-  // TODO(xians): Check if the OS has the device with |device_id_|, return
-  // |SPEECH_AUDIO_ERROR_DETAILS_NO_MIC| if the target device does not exist.
-  if (!audio_manager->HasAudioInputDevices()) {
+  int chunk_duration_ms = recognition_engine_->GetDesiredAudioChunkDurationMs();
+
+  if (!device_params_.IsValid()) {
+    DLOG(ERROR) << "Audio input device not found";
     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE,
                                         SPEECH_AUDIO_ERROR_DETAILS_NO_MIC));
   }
 
-  int chunk_duration_ms = recognition_engine_->GetDesiredAudioChunkDurationMs();
-
-  AudioParameters in_params = audio_manager->GetInputStreamParameters(
-      device_id_);
-  if (!in_params.IsValid() && !unit_test_is_active) {
-    DLOG(ERROR) << "Invalid native audio input parameters";
-    return Abort(
-        SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE));
-  }
-
   // Audio converter shall provide audio based on these parameters as output.
   // Hard coded, WebSpeech specific parameters are utilized here.
   int frames_per_buffer = (kAudioSampleRate * chunk_duration_ms) / 1000;
   AudioParameters output_parameters = AudioParameters(
       AudioParameters::AUDIO_PCM_LOW_LATENCY, kChannelLayout, kAudioSampleRate,
       kNumBitsPerAudioSample, frames_per_buffer);
   DVLOG(1) << "SRI::output_parameters: "
            << output_parameters.AsHumanReadableString();
 
   // Audio converter will receive audio based on these parameters as input.
   // On Windows we start by verifying that Core Audio is supported. If not,
   // the WaveIn API is used and we might as well avoid all audio conversations
   // since WaveIn does the conversion for us.
   // TODO(henrika): this code should be moved to platform dependent audio
   // managers.
   bool use_native_audio_params = true;
 #if defined(OS_WIN)
   use_native_audio_params = media::CoreAudioUtil::IsSupported();
   DVLOG_IF(1, !use_native_audio_params) << "Reverting to WaveIn for WebSpeech";
 #endif
 
   AudioParameters input_parameters = output_parameters;
-  if (use_native_audio_params && !unit_test_is_active) {
+
+  // AUDIO_FAKE means we are running a test.
+  if (use_native_audio_params &&
+      device_params_.format() != media::AudioParameters::AUDIO_FAKE) {
     // Use native audio parameters but avoid opening up at the native buffer
     // size. Instead use same frame size (in milliseconds) as WebSpeech uses.
     // We rely on internal buffers in the audio back-end to fulfill this request
     // and the idea is to simplify the audio conversion since each Convert()
     // call will then render exactly one ProvideInput() call.
-    // in_params.sample_rate()
-    input_parameters = in_params;
+    input_parameters = device_params_;
     frames_per_buffer =
-        ((in_params.sample_rate() * chunk_duration_ms) / 1000.0) + 0.5;
+        ((input_parameters.sample_rate() * chunk_duration_ms) / 1000.0) + 0.5;
     input_parameters.set_frames_per_buffer(frames_per_buffer);
     DVLOG(1) << "SRI::input_parameters: "
              << input_parameters.AsHumanReadableString();
   }
 
   // Create an audio converter which converts data between native input format
   // and WebSpeech specific output format.
   audio_converter_.reset(
       new OnDataConverter(input_parameters, output_parameters));
 
   audio_controller_ = AudioInputController::Create(
-      audio_manager, this, this, nullptr, nullptr, input_parameters, device_id_,
+      GetAudioSystem()->GetAudioManager(), this, this, nullptr, nullptr,
+      input_parameters, device_id_,
       /*agc_is_enabled*/ false);
 
   if (!audio_controller_.get()) {
     return Abort(
         SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE));
   }
 
   audio_log_->OnCreated(0, input_parameters, device_id_);
 
   // The endpointer needs to estimate the environment/background noise before
@@ skipping 79 matching lines @@
     return Abort(
         SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE));
   } else if (event_args.event == EVENT_ENGINE_ERROR) {
     return Abort(event_args.engine_error);
   }
   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_ABORTED));
 }
 
 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::Abort(
     const SpeechRecognitionError& error) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
   if (IsCapturingAudio())
     CloseAudioControllerAsynchronously();
 
   DVLOG(1) << "SpeechRecognizerImpl canceling recognition. ";
 
+  if (state_ == STATE_PREPARING) {
+    // Cancel an outstanding reply from AudioSystem.
+    weak_ptr_factory_.InvalidateWeakPtrs();
+  }
+
   // The recognition engine is initialized only after STATE_STARTING.
   if (state_ > STATE_STARTING) {
     DCHECK(recognition_engine_.get() != NULL);
     recognition_engine_->EndRecognition();
   }
 
   if (state_ > STATE_WAITING_FOR_SPEECH && state_ < STATE_WAITING_FINAL_RESULT)
     listener()->OnSoundEnd(session_id());
 
   if (state_ > STATE_STARTING && state_ < STATE_WAITING_FINAL_RESULT)
@@ skipping 116 matching lines @@
 
   float noise_level = (endpointer_.NoiseLevelDb() - kAudioMeterMinDb) /
       (kAudioMeterDbRange / kAudioMeterRangeMaxUnclipped);
   noise_level = std::min(std::max(0.0f, noise_level),
                          kAudioMeterRangeMaxUnclipped);
 
   listener()->OnAudioLevelsChange(
       session_id(), clip_detected ? 1.0f : audio_level_, noise_level);
 }
 
-void SpeechRecognizerImpl::SetAudioManagerForTesting(
-    AudioManager* audio_manager) {
-  audio_manager_for_tests_ = audio_manager;
+void SpeechRecognizerImpl::SetAudioSystemForTesting(
+    media::AudioSystem* audio_system) {
+  audio_system_for_tests_ = audio_system;
+}
+
+media::AudioSystem* SpeechRecognizerImpl::GetAudioSystem() {
+  return audio_system_for_tests_ ? audio_system_for_tests_ : audio_system_;
 }
 
 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(FSMEvent event_value)
     : event(event_value),
       audio_data(NULL),
       engine_error(SPEECH_RECOGNITION_ERROR_NONE) {
 }
 
 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(const FSMEventArgs& other) =
     default;
 
 SpeechRecognizerImpl::FSMEventArgs::~FSMEventArgs() {
 }
 
 }  // namespace content