Switch Speech Recognition to asynchronous callback-based AudioManager interactions.

content/browser/speech/speech_recognizer_impl.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/speech/speech_recognizer_impl.h"
 
 #include <stdint.h>
 
 #include "base/bind.h"
 #include "base/macros.h"
 #include "base/time/time.h"
 #include "build/build_config.h"
 #include "content/browser/browser_main_loop.h"
 #include "content/browser/media/media_internals.h"
 #include "content/browser/speech/audio_buffer.h"
 #include "content/public/browser/speech_recognition_event_listener.h"
+#include "media/audio/audio_manager.h"
+#include "media/audio/audio_system.h"
 #include "media/base/audio_converter.h"
 
 #if defined(OS_WIN)
 #include "media/audio/win/core_audio_util_win.h"
 #endif
 
 using media::AudioBus;
 using media::AudioConverter;
 using media::AudioInputController;
 using media::AudioManager;
@@ skipping 75 matching lines @@
 }
 
 }  // namespace
 
 const int SpeechRecognizerImpl::kAudioSampleRate = 16000;
 const ChannelLayout SpeechRecognizerImpl::kChannelLayout =
     media::CHANNEL_LAYOUT_MONO;
 const int SpeechRecognizerImpl::kNumBitsPerAudioSample = 16;
 const int SpeechRecognizerImpl::kNoSpeechTimeoutMs = 8000;
 const int SpeechRecognizerImpl::kEndpointerEstimationTimeMs = 300;
-media::AudioManager* SpeechRecognizerImpl::audio_manager_for_tests_ = NULL;
+media::AudioSystem* SpeechRecognizerImpl::audio_system_for_tests_ = nullptr;
 
 static_assert(SpeechRecognizerImpl::kNumBitsPerAudioSample % 8 == 0,
               "kNumBitsPerAudioSample must be a multiple of 8");
 
 // SpeechRecognizerImpl::OnDataConverter implementation
 
 SpeechRecognizerImpl::OnDataConverter::OnDataConverter(
     const AudioParameters& input_params,
     const AudioParameters& output_params)
     : audio_converter_(input_params, output_params, false),
@@ skipping 43 matching lines @@
   input_bus_->CopyTo(dest);
   // Indicate that the recorded audio has in fact been used by the converter.
   data_was_converted_ = true;
   return 1;
 }
 
 // SpeechRecognizerImpl implementation
 
 SpeechRecognizerImpl::SpeechRecognizerImpl(
     SpeechRecognitionEventListener* listener,
+    media::AudioSystem* audio_system,
     int session_id,
     bool continuous,
     bool provisional_results,
     SpeechRecognitionEngine* engine)
     : SpeechRecognizer(listener, session_id),
+      audio_system_(audio_system),
       recognition_engine_(engine),
       endpointer_(kAudioSampleRate),
       audio_log_(MediaInternals::GetInstance()->CreateAudioLog(
           media::AudioLogFactory::AUDIO_INPUT_CONTROLLER)),
       is_dispatching_event_(false),
       provisional_results_(provisional_results),
       end_of_utterance_(false),
-      state_(STATE_IDLE) {
-  DCHECK(recognition_engine_ != NULL);
+      state_(STATE_IDLE),
+      weak_ptr_factory_(this) {
+  DCHECK(recognition_engine_ != nullptr);
+  DCHECK(audio_system_ != nullptr);
   if (!continuous) {
     // In single shot (non-continous) recognition,
     // the session is automatically ended after:
     //  - 0.5 seconds of silence if time <  3 seconds
     //  - 1   seconds of silence if time >= 3 seconds
     endpointer_.set_speech_input_complete_silence_length(
         base::Time::kMicrosecondsPerSecond / 2);
     endpointer_.set_long_speech_input_complete_silence_length(
         base::Time::kMicrosecondsPerSecond);
     endpointer_.set_long_speech_length(3 * base::Time::kMicrosecondsPerSecond);
@@ skipping 13 matching lines @@
 // NOTE:all the external events and requests should be enqueued (PostTask), even
 // if they come from the same (IO) thread, in order to preserve the relationship
 // of causality between events and avoid interleaved event processing due to
 // synchronous callbacks.
 
 void SpeechRecognizerImpl::StartRecognition(const std::string& device_id) {
   DCHECK(!device_id.empty());
   device_id_ = device_id;
 
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
-                          base::Bind(&SpeechRecognizerImpl::DispatchEvent,
-                                     this, FSMEventArgs(EVENT_START)));
+                          base::Bind(&SpeechRecognizerImpl::DispatchEvent, this,
+                                     FSMEventArgs(EVENT_PREPARE)));
 }
 
 void SpeechRecognizerImpl::AbortRecognition() {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
                                      this, FSMEventArgs(EVENT_ABORT)));
 }
 
 void SpeechRecognizerImpl::StopAudioCapture() {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
@@ skipping 131 matching lines @@
 SpeechRecognizerImpl::ExecuteTransitionAndGetNextState(
     const FSMEventArgs& event_args) {
   const FSMEvent event = event_args.event;
   switch (state_) {
     case STATE_IDLE:
       switch (event) {
         // TODO(primiano): restore UNREACHABLE_CONDITION on EVENT_ABORT and
         // EVENT_STOP_CAPTURE below once speech input extensions are fixed.
         case EVENT_ABORT:
           return AbortSilently(event_args);
+        case EVENT_PREPARE:
+          return PrepareRecognition(event_args);
+        case EVENT_START:
+          return NotFeasible(event_args);
+        case EVENT_STOP_CAPTURE:
+          return AbortSilently(event_args);
+        case EVENT_AUDIO_DATA:     // Corner cases related to queued messages
+        case EVENT_ENGINE_RESULT:  // being lately dispatched.
+        case EVENT_ENGINE_ERROR:
+        case EVENT_AUDIO_ERROR:
+          return DoNothing(event_args);
+      }
+      break;
+    case STATE_PREPARING:
+      switch (event) {
+        case EVENT_ABORT:
+          return AbortSilently(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return StartRecording(event_args);
         case EVENT_STOP_CAPTURE:
           return AbortSilently(event_args);
         case EVENT_AUDIO_DATA:     // Corner cases related to queued messages
         case EVENT_ENGINE_RESULT:  // being lately dispatched.
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return DoNothing(event_args);
       }
       break;
     case STATE_STARTING:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return AbortSilently(event_args);
         case EVENT_AUDIO_DATA:
           return StartRecognitionEngine(event_args);
         case EVENT_ENGINE_RESULT:
           return NotFeasible(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_ESTIMATING_ENVIRONMENT:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return StopCaptureAndWaitForResult(event_args);
         case EVENT_AUDIO_DATA:
           return WaitEnvironmentEstimationCompletion(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessIntermediateResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_WAITING_FOR_SPEECH:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return StopCaptureAndWaitForResult(event_args);
         case EVENT_AUDIO_DATA:
           return DetectUserSpeechOrTimeout(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessIntermediateResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_RECOGNIZING:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
           return StopCaptureAndWaitForResult(event_args);
         case EVENT_AUDIO_DATA:
           return DetectEndOfSpeech(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessIntermediateResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
       }
       break;
     case STATE_WAITING_FINAL_RESULT:
       switch (event) {
         case EVENT_ABORT:
           return AbortWithError(event_args);
+        case EVENT_PREPARE:
+          return NotFeasible(event_args);
         case EVENT_START:
           return NotFeasible(event_args);
         case EVENT_STOP_CAPTURE:
         case EVENT_AUDIO_DATA:
           return DoNothing(event_args);
         case EVENT_ENGINE_RESULT:
           return ProcessFinalResult(event_args);
         case EVENT_ENGINE_ERROR:
         case EVENT_AUDIO_ERROR:
           return AbortWithError(event_args);
@@ skipping 36 matching lines @@
   if (route_to_vumeter) {
     DCHECK(route_to_endpointer);  // Depends on endpointer due to |rms|.
     UpdateSignalAndNoiseLevels(rms, clip_detected);
   }
   if (route_to_sr_engine) {
     DCHECK(recognition_engine_.get() != NULL);
     recognition_engine_->TakeAudioChunk(raw_audio);
   }
 }
 
+void SpeechRecognizerImpl::OnDeviceInfo(const media::AudioParameters& params) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  device_params_ = params;
+  DVLOG(1) << "Device parameters: " << device_params_.AsHumanReadableString();
+  DispatchEvent(FSMEventArgs(EVENT_START));
+}
+
+SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::PrepareRecognition(
+    const FSMEventArgs&) {
+  DCHECK(state_ == STATE_IDLE);
+  DCHECK(recognition_engine_.get() != NULL);
+  DCHECK(!IsCapturingAudio());
+  GetAudioSystem()->GetInputStreamParameters(

[tommi (sloooow) - chröme, 2017/02/02 16:27:18]

Instead of adding AudioSystem etc, could you post to the audio thread here, call
GetInputParameters() and post pack to the IO thread when you're done to complete
the state transition?  Maybe I'm missing something but it feels like we're
adding a lot of complexity that we can avoid adding.

+      device_id_, base::Bind(&SpeechRecognizerImpl::OnDeviceInfo,
+                             weak_ptr_factory_.GetWeakPtr()));
+
+  listener()->OnRecognitionStart(session_id());
+  return STATE_PREPARING;
+}
+
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
-  DCHECK(state_ == STATE_IDLE);
+  DCHECK(state_ == STATE_PREPARING);
   DCHECK(recognition_engine_.get() != NULL);
   DCHECK(!IsCapturingAudio());
-  const bool unit_test_is_active = (audio_manager_for_tests_ != NULL);
-  AudioManager* audio_manager = unit_test_is_active ?
-                                audio_manager_for_tests_ :
-                                AudioManager::Get();
-  DCHECK(audio_manager != NULL);
 
   DVLOG(1) << "SpeechRecognizerImpl starting audio capture.";
   num_samples_recorded_ = 0;
   audio_level_ = 0;
   end_of_utterance_ = false;
-  listener()->OnRecognitionStart(session_id());
 
-  // TODO(xians): Check if the OS has the device with |device_id_|, return
-  // |SPEECH_AUDIO_ERROR_DETAILS_NO_MIC| if the target device does not exist.
-  if (!audio_manager->HasAudioInputDevices()) {
+  int chunk_duration_ms = recognition_engine_->GetDesiredAudioChunkDurationMs();
+
+  if (!device_params_.IsValid()) {
+    DLOG(ERROR) << "Audio input device not found";
     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE,
                                         SPEECH_AUDIO_ERROR_DETAILS_NO_MIC));
   }
 
-  int chunk_duration_ms = recognition_engine_->GetDesiredAudioChunkDurationMs();
-
-  AudioParameters in_params = audio_manager->GetInputStreamParameters(
-      device_id_);
-  if (!in_params.IsValid() && !unit_test_is_active) {
-    DLOG(ERROR) << "Invalid native audio input parameters";
-    return Abort(
-        SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE));
-  }
-
   // Audio converter shall provide audio based on these parameters as output.
   // Hard coded, WebSpeech specific parameters are utilized here.
   int frames_per_buffer = (kAudioSampleRate * chunk_duration_ms) / 1000;
   AudioParameters output_parameters = AudioParameters(
       AudioParameters::AUDIO_PCM_LOW_LATENCY, kChannelLayout, kAudioSampleRate,
       kNumBitsPerAudioSample, frames_per_buffer);
   DVLOG(1) << "SRI::output_parameters: "
            << output_parameters.AsHumanReadableString();
 
   // Audio converter will receive audio based on these parameters as input.
   // On Windows we start by verifying that Core Audio is supported. If not,
   // the WaveIn API is used and we might as well avoid all audio conversations
   // since WaveIn does the conversion for us.
   // TODO(henrika): this code should be moved to platform dependent audio
   // managers.
   bool use_native_audio_params = true;
 #if defined(OS_WIN)
   use_native_audio_params = media::CoreAudioUtil::IsSupported();
   DVLOG_IF(1, !use_native_audio_params) << "Reverting to WaveIn for WebSpeech";
 #endif
 
   AudioParameters input_parameters = output_parameters;
-  if (use_native_audio_params && !unit_test_is_active) {
+
+  // AUDIO_FAKE means we are running a test.
+  if (use_native_audio_params &&
+      device_params_.format() != media::AudioParameters::AUDIO_FAKE) {
     // Use native audio parameters but avoid opening up at the native buffer
     // size. Instead use same frame size (in milliseconds) as WebSpeech uses.
     // We rely on internal buffers in the audio back-end to fulfill this request
     // and the idea is to simplify the audio conversion since each Convert()
     // call will then render exactly one ProvideInput() call.
-    // in_params.sample_rate()
-    input_parameters = in_params;
+    input_parameters = device_params_;
     frames_per_buffer =
-        ((in_params.sample_rate() * chunk_duration_ms) / 1000.0) + 0.5;
+        ((input_parameters.sample_rate() * chunk_duration_ms) / 1000.0) + 0.5;
     input_parameters.set_frames_per_buffer(frames_per_buffer);
     DVLOG(1) << "SRI::input_parameters: "
              << input_parameters.AsHumanReadableString();
   }
 
   // Create an audio converter which converts data between native input format
   // and WebSpeech specific output format.
   audio_converter_.reset(
       new OnDataConverter(input_parameters, output_parameters));
 
-  audio_controller_ = AudioInputController::Create(
-      audio_manager, this, this, input_parameters, device_id_, NULL);
+  audio_controller_ =
+      AudioInputController::Create(GetAudioSystem()->GetAudioManager(), this,
+                                   this, input_parameters, device_id_, NULL);
 
   if (!audio_controller_.get()) {
     return Abort(
         SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE));
   }
 
   audio_log_->OnCreated(0, input_parameters, device_id_);
 
   // The endpointer needs to estimate the environment/background noise before
   // starting to treat the audio as user input. We wait in the state
@@ skipping 78 matching lines @@
     return Abort(
         SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO_CAPTURE));
   } else if (event_args.event == EVENT_ENGINE_ERROR) {
     return Abort(event_args.engine_error);
   }
   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_ABORTED));
 }
 
 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::Abort(
     const SpeechRecognitionError& error) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
   if (IsCapturingAudio())
     CloseAudioControllerAsynchronously();
 
   DVLOG(1) << "SpeechRecognizerImpl canceling recognition. ";
 
+  if (state_ == STATE_PREPARING) {
+    // Cancel an outstanding reply from AudioSystem.
+    weak_ptr_factory_.InvalidateWeakPtrs();
+  }
+
   // The recognition engine is initialized only after STATE_STARTING.
   if (state_ > STATE_STARTING) {
     DCHECK(recognition_engine_.get() != NULL);
     recognition_engine_->EndRecognition();
   }
 
   if (state_ > STATE_WAITING_FOR_SPEECH && state_ < STATE_WAITING_FINAL_RESULT)
     listener()->OnSoundEnd(session_id());
 
   if (state_ > STATE_STARTING && state_ < STATE_WAITING_FINAL_RESULT)
@@ skipping 116 matching lines @@
 
   float noise_level = (endpointer_.NoiseLevelDb() - kAudioMeterMinDb) /
       (kAudioMeterDbRange / kAudioMeterRangeMaxUnclipped);
   noise_level = std::min(std::max(0.0f, noise_level),
                          kAudioMeterRangeMaxUnclipped);
 
   listener()->OnAudioLevelsChange(
       session_id(), clip_detected ? 1.0f : audio_level_, noise_level);
 }
 
-void SpeechRecognizerImpl::SetAudioManagerForTesting(
-    AudioManager* audio_manager) {
-  audio_manager_for_tests_ = audio_manager;
+void SpeechRecognizerImpl::SetAudioSystemForTesting(
+    media::AudioSystem* audio_system) {
+  audio_system_for_tests_ = audio_system;
+}
+
+media::AudioSystem* SpeechRecognizerImpl::GetAudioSystem() {
+  return audio_system_for_tests_ ? audio_system_for_tests_ : audio_system_;
 }
 
 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(FSMEvent event_value)
     : event(event_value),
       audio_data(NULL),
       engine_error(SPEECH_RECOGNITION_ERROR_NONE) {
 }
 
 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(const FSMEventArgs& other) =
     default;
 
 SpeechRecognizerImpl::FSMEventArgs::~FSMEventArgs() {
 }
 
 }  // namespace content