Example use case for AudioSharedBuffer

content/browser/speech/speech_recognizer_impl.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/speech/speech_recognizer_impl.h"
 
 #include "base/basictypes.h"
 #include "base/bind.h"
+#include "base/memory/shared_memory.h"
+#include "base/process/process_handle.h"
+#include "base/threading/thread.h"
 #include "base/time/time.h"
 #include "content/browser/browser_main_loop.h"
 #include "content/browser/media/media_internals.h"
 #include "content/browser/speech/audio_buffer.h"
 #include "content/browser/speech/google_one_shot_remote_engine.h"
+#include "content/public/browser/browser_message_filter.h"
 #include "content/public/browser/speech_recognition_event_listener.h"
 #include "media/base/audio_converter.h"
 #include "net/url_request/url_request_context_getter.h"
 
 #if defined(OS_WIN)
 #include "media/audio/win/core_audio_util_win.h"
 #endif
 
 using media::AudioBus;
 using media::AudioConverter;
+using media::AudioInputBufferParameters;
 using media::AudioInputController;
 using media::AudioManager;
 using media::AudioParameters;
 using media::ChannelLayout;
 
 namespace content {
 
 // Private class which encapsulates the audio converter and the
 // AudioConverter::InputCallback. It handles resampling, buffering and
 // channel mixing between input and output parameters.
 class SpeechRecognizerImpl::OnDataConverter
     : public media::AudioConverter::InputCallback {
  public:
   OnDataConverter(const AudioParameters& input_params,
                   const AudioParameters& output_params);
   virtual ~OnDataConverter();
 
   // Converts input audio |data| bus into an AudioChunk where the input format
   // is given by |input_parameters_| and the output format by
   // |output_parameters_|.
+  // Skips conversion on |audio_converter_| when input and output formats match.
   scoped_refptr<AudioChunk> Convert(const AudioBus* data);
 
  private:
   // media::AudioConverter::InputCallback implementation.
   virtual double ProvideInput(AudioBus* dest,
                               base::TimeDelta buffer_delay) OVERRIDE;
 
   // Handles resampling, buffering, and channel mixing between input and output
   // parameters.
   AudioConverter audio_converter_;
 
   scoped_ptr<AudioBus> input_bus_;
   scoped_ptr<AudioBus> output_bus_;
   const AudioParameters input_parameters_;
   const AudioParameters output_parameters_;
   bool waiting_for_input_;
+  bool need_conversion_;
   scoped_ptr<uint8[]> converted_data_;
 
   DISALLOW_COPY_AND_ASSIGN(OnDataConverter);
 };
 
 namespace {
 
 // The following constants are related to the volume level indicator shown in
 // the UI for recorded audio.
 // Multiplier used when new volume is greater than previous level.
@@ skipping 47 matching lines @@
 SpeechRecognizerImpl::OnDataConverter::OnDataConverter(
     const AudioParameters& input_params, const AudioParameters& output_params)
     : audio_converter_(input_params, output_params, false),
       input_bus_(AudioBus::Create(input_params)),
       output_bus_(AudioBus::Create(output_params)),
       input_parameters_(input_params),
       output_parameters_(output_params),
       waiting_for_input_(false),
       converted_data_(new uint8[output_parameters_.GetBytesPerBuffer()]) {
   audio_converter_.AddInput(this);
+  need_conversion_ = !(input_parameters_ == output_parameters_);
 }
 
 SpeechRecognizerImpl::OnDataConverter::~OnDataConverter() {
   // It should now be safe to unregister the converter since no more OnData()
   // callbacks are outstanding at this point.
   audio_converter_.RemoveInput(this);
 }
 
 scoped_refptr<AudioChunk> SpeechRecognizerImpl::OnDataConverter::Convert(
     const AudioBus* data) {
   CHECK_EQ(data->frames(), input_parameters_.frames_per_buffer());
 
-  data->CopyTo(input_bus_.get());
-
-  waiting_for_input_ = true;
-  audio_converter_.Convert(output_bus_.get());
+  // Do conversion if needed - only when |input_parameters_| differ from
+  // |output_parameters_|.
+  if (need_conversion_) {
+    data->CopyTo(input_bus_.get());
+    waiting_for_input_ = true;
+    audio_converter_.Convert(output_bus_.get());
+  } else {
+    data->CopyTo(output_bus_.get());
+  }
 
   output_bus_->ToInterleaved(
       output_bus_->frames(), output_parameters_.bits_per_sample() / 8,
       converted_data_.get());
 
   // TODO(primiano): Refactor AudioChunk to avoid the extra-copy here
   // (see http://crbug.com/249316 for details).
   return scoped_refptr<AudioChunk>(new AudioChunk(
       converted_data_.get(),
       output_parameters_.GetBytesPerBuffer(),
@@ skipping 15 matching lines @@
   return 1;
 }
 
 // SpeechRecognizerImpl implementation
 
 SpeechRecognizerImpl::SpeechRecognizerImpl(
     SpeechRecognitionEventListener* listener,
     int session_id,
     bool continuous,
     bool provisional_results,
+    bool using_audio_track,
+    base::ProcessHandle peer_process_handle,
     SpeechRecognitionEngine* engine)
     : SpeechRecognizer(listener, session_id),
       recognition_engine_(engine),
       endpointer_(kAudioSampleRate),
       audio_log_(MediaInternals::GetInstance()->CreateAudioLog(
           media::AudioLogFactory::AUDIO_INPUT_CONTROLLER)),
       is_dispatching_event_(false),
       provisional_results_(provisional_results),
+      using_audio_track_(using_audio_track),
+      peer_process_handle_(peer_process_handle),
       state_(STATE_IDLE) {
   DCHECK(recognition_engine_ != NULL);
   if (!continuous) {
     // In single shot (non-continous) recognition,
     // the session is automatically ended after:
     //  - 0.5 seconds of silence if time <  3 seconds
     //  - 1   seconds of silence if time >= 3 seconds
     endpointer_.set_speech_input_complete_silence_length(
         base::Time::kMicrosecondsPerSecond / 2);
     endpointer_.set_long_speech_input_complete_silence_length(
@@ skipping 11 matching lines @@
 }
 
 // -------  Methods that trigger Finite State Machine (FSM) events ------------
 
 // NOTE:all the external events and requests should be enqueued (PostTask), even
 // if they come from the same (IO) thread, in order to preserve the relationship
 // of causality between events and avoid interleaved event processing due to
 // synchronous callbacks.
 
 void SpeechRecognizerImpl::StartRecognition(const std::string& device_id) {
-  DCHECK(!device_id.empty());
+  DCHECK(!device_id.empty() ^ using_audio_track_);
   device_id_ = device_id;
 
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
                                      this, FSMEventArgs(EVENT_START)));
 }
 
 void SpeechRecognizerImpl::AbortRecognition() {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
@@ skipping 10 matching lines @@
   // Checking the FSM state from another thread (thus, while the FSM is
   // potentially concurrently evolving) is meaningless.
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   return state_ != STATE_IDLE && state_ != STATE_ENDED;
 }
 
 bool SpeechRecognizerImpl::IsCapturingAudio() const {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); // See IsActive().
   const bool is_capturing_audio = state_ >= STATE_STARTING &&
                                   state_ <= STATE_RECOGNIZING;
-  DCHECK((is_capturing_audio && (audio_controller_.get() != NULL)) ||
+  DCHECK((is_capturing_audio &&
+          (using_audio_track_ || (audio_controller_.get() != NULL))) ||
          (!is_capturing_audio && audio_controller_.get() == NULL));
   return is_capturing_audio;
 }
 
 const SpeechRecognitionEngine&
 SpeechRecognizerImpl::recognition_engine() const {
   return *(recognition_engine_.get());
 }
 
 SpeechRecognizerImpl::~SpeechRecognizerImpl() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   endpointer_.EndSession();
   if (audio_controller_.get()) {
     audio_controller_->Close(
         base::Bind(&KeepAudioControllerRefcountedForDtor, audio_controller_));
     audio_log_->OnClosed(0);
   }
+
+  if (audio_sync_thread_.get()) {
+    base::ThreadRestrictions::SetIOAllowed(true);
+    audio_sync_socket_->Shutdown();
+    audio_sync_thread_->Stop();
+  }
 }
 
 // Invoked in the audio thread.
 void SpeechRecognizerImpl::OnError(AudioInputController* controller,
     media::AudioInputController::ErrorCode error_code) {
   FSMEventArgs event_args(EVENT_AUDIO_ERROR);
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
                                      this, event_args));
 }
@@ skipping 199 matching lines @@
                                 state_ <= STATE_RECOGNIZING;
   const bool clip_detected = DetectClipping(raw_audio);
   float rms = 0.0f;
 
   num_samples_recorded_ += raw_audio.NumSamples();
 
   if (route_to_endpointer)
     endpointer_.ProcessAudio(raw_audio, &rms);
 
   if (route_to_vumeter) {
-    DCHECK(route_to_endpointer); // Depends on endpointer due to |rms|.
+    DCHECK(route_to_endpointer);  // Depends on endpointer due to |rms|.
     UpdateSignalAndNoiseLevels(rms, clip_detected);
   }
   if (route_to_sr_engine) {
     DCHECK(recognition_engine_.get() != NULL);
     recognition_engine_->TakeAudioChunk(raw_audio);
   }
 }
 
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
@@ skipping 26 matching lines @@
     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
   }
 
   // Audio converter shall provide audio based on these parameters as output.
   // Hard coded, WebSpeech specific parameters are utilized here.
   int frames_per_buffer = (kAudioSampleRate * chunk_duration_ms) / 1000;
   AudioParameters output_parameters = AudioParameters(
       AudioParameters::AUDIO_PCM_LOW_LATENCY, kChannelLayout, kAudioSampleRate,
       kNumBitsPerAudioSample, frames_per_buffer);
 
+  // Check if using audio track as source (audio data processed on renderer).
+  // If |StartRecordingUsingAudioTrack()| fails, we abort the recognition.
+  if (using_audio_track_)
+    return StartRecordingUsingAudioTrack(output_parameters)
+               ? STATE_STARTING
+               : Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
+
+  // Get audio data from microphone on the browser.
+
   // Audio converter will receive audio based on these parameters as input.
   // On Windows we start by verifying that Core Audio is supported. If not,
   // the WaveIn API is used and we might as well avoid all audio conversations
   // since WaveIn does the conversion for us.
   // TODO(henrika): this code should be moved to platform dependent audio
   // managers.
   bool use_native_audio_params = true;
 #if defined(OS_WIN)
   use_native_audio_params = media::CoreAudioUtil::IsSupported();
   DVLOG_IF(1, !use_native_audio_params) << "Reverting to WaveIn for WebSpeech";
@@ skipping 36 matching lines @@
   // The endpointer needs to estimate the environment/background noise before
   // starting to treat the audio as user input. We wait in the state
   // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching
   // to user input mode.
   endpointer_.SetEnvironmentEstimationMode();
   audio_controller_->Record();
   audio_log_->OnStarted(0);
   return STATE_STARTING;
 }
 
+bool SpeechRecognizerImpl::StartRecordingUsingAudioTrack(
+    const media::AudioParameters& params) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  // Create the shared memory and share with the renderer process.
+  uint32 shared_memory_size = sizeof(uint32) +
+                              AudioBus::CalculateMemorySize(params);
+  shared_memory_.reset(new base::SharedMemory());
+  if (!shared_memory_->CreateAndMapAnonymous(shared_memory_size)) {
+    DLOG(ERROR) << "Failed to create shared memory for the speech recognizer";
+    return false;
+  }
+  base::SharedMemoryHandle foreign_memory_handle;
+  if (!shared_memory_->ShareToProcess(base::GetCurrentProcessHandle(),
+                                      &foreign_memory_handle)) {
+    DLOG(ERROR) << "Failed to map and share the shared memory";
+    return false;
+  }
+  media::AudioSharedBuffer* buffer =
+      static_cast<media::AudioSharedBuffer*>(shared_memory_->memory());
+  audio_track_bus_ = AudioBus::WrapMemory(params, buffer->audio);
+  // Use |buffer->params.size| from shared memory to synchronize by counting
+  // audio buffers when receiving audio data from the renderer.
+  buffer_index_ = &(buffer->buffer_index);
+  *buffer_index_ = 0U;
+  // Reusing the audio converter only to create chunks from the bus.
+  // Same input and output format indicate no additional conversion here.
+  audio_converter_.reset(new OnDataConverter(params, params));
+
+  // The endpointer needs to estimate the environment/background noise before
+  // starting to treat the audio as user input. We wait in the state
+  // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching
+  // to user input mode.
+  endpointer_.SetEnvironmentEstimationMode();
+
+  foreign_sync_socket_.reset(new base::CancelableSyncSocket());
+  audio_sync_socket_.reset(new base::CancelableSyncSocket());
+  if (!base::SyncSocket::CreatePair(audio_sync_socket_.get(),
+                                    foreign_sync_socket_.get())) {
+    DLOG(ERROR) << "Cannot create SyncSocket pair";
+    return false;
+  }
+
+  base::SyncSocket::TransitDescriptor foreign_socket_descriptor;
+  if (!foreign_sync_socket_->PrepareTransitDescriptor(
+          peer_process_handle_, &foreign_socket_descriptor)) {
+    DLOG(ERROR) << "Cannot prepare syncsocket transit descriptor";
+    return false;
+  }
+
+  // The thread check will be first invoked in |OnAudioTrackData()|.
+  audio_sync_thread_checker_.DetachFromThread();
+  audio_sync_thread_.reset(
+      new base::Thread("SpeechRecognitionAudioTrackThread"));
+  audio_sync_thread_->Start();
+
+  DCHECK_EQ(0U, *buffer_index_);
+  if (!audio_sync_thread_->message_loop_proxy()
+           ->PostTask(FROM_HERE,
+                      base::Bind(&SpeechRecognizerImpl::OnAudioTrackData,
+                                 base::Unretained(this)))) {
+    DLOG(ERROR) << "Cannot post audio sync thread task";
+    return false;
+  }
+
+  listener()->OnAudioTrackReady(session_id(), params, foreign_memory_handle,
+                                foreign_socket_descriptor);
+
+  return true;
+}
+
+void SpeechRecognizerImpl::OnAudioTrackData() {
+  DCHECK(!BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(audio_sync_thread_checker_.CalledOnValidThread());
+  base::ThreadRestrictions::SetIOAllowed(true);
+
+  // TODO(burnik): We should receive with a timeout here. And when it occurs,
+  // we should stop the recognition assuming there is no more audio data.
+  size_t bytes_received =
+      audio_sync_socket_->Receive(buffer_index_, sizeof(*buffer_index_));
+
+  if (bytes_received != sizeof(*buffer_index_)) {
+    DCHECK_EQ(bytes_received, 0U);
+    return;
+  }
+
+  FSMEventArgs event_args(EVENT_AUDIO_DATA);
+  // Convert audio from native format to fixed format used by WebSpeech.
+  event_args.audio_data = audio_converter_->Convert(audio_track_bus_.get());
+  // Process on browser thread.
+  BrowserThread::PostTask(
+      BrowserThread::IO, FROM_HERE,
+      base::Bind(&SpeechRecognizerImpl::DispatchEvent, this, event_args));
+
+  // Notify host that the buffer was consumed via shared memory.
+  (*buffer_index_)++;
+
+  // Post this task for next round of data.
+  audio_sync_thread_->message_loop_proxy()->PostTask(
+      FROM_HERE, base::Bind(&SpeechRecognizerImpl::OnAudioTrackData,
+                            base::Unretained(this)));
+}
+
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::StartRecognitionEngine(const FSMEventArgs& event_args) {
   // This is the first audio packet captured, so the recognition engine is
   // started and the delegate notified about the event.
   DCHECK(recognition_engine_.get() != NULL);
   recognition_engine_->StartRecognition();
   listener()->OnAudioStart(session_id());
 
   // This is a little hack, since TakeAudioChunk() is already called by
   // ProcessAudioPipeline(). It is the best tradeoff, unless we allow dropping
@@ skipping 30 matching lines @@
   if (endpointer_.speech_input_complete())
     return StopCaptureAndWaitForResult(event_args);
   return STATE_RECOGNIZING;
 }
 
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::StopCaptureAndWaitForResult(const FSMEventArgs&) {
   DCHECK(state_ >= STATE_ESTIMATING_ENVIRONMENT && state_ <= STATE_RECOGNIZING);
 
   DVLOG(1) << "Concluding recognition";
-  CloseAudioControllerAsynchronously();
+  if (!using_audio_track_) CloseAudioControllerAsynchronously();
   recognition_engine_->AudioChunksEnded();
 
   if (state_ > STATE_WAITING_FOR_SPEECH)
     listener()->OnSoundEnd(session_id());
 
   listener()->OnAudioEnd(session_id());
   return STATE_WAITING_FINAL_RESULT;
 }
 
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::AbortSilently(const FSMEventArgs& event_args) {
   DCHECK_NE(event_args.event, EVENT_AUDIO_ERROR);
   DCHECK_NE(event_args.event, EVENT_ENGINE_ERROR);
   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NONE));
 }
 
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::AbortWithError(const FSMEventArgs& event_args) {
   if (event_args.event == EVENT_AUDIO_ERROR) {
     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
   } else if (event_args.event == EVENT_ENGINE_ERROR) {
     return Abort(event_args.engine_error);
   }
   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_ABORTED));
 }
 
 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::Abort(
     const SpeechRecognitionError& error) {
-  if (IsCapturingAudio())
+  if (IsCapturingAudio() && !using_audio_track_)
     CloseAudioControllerAsynchronously();
 
   DVLOG(1) << "SpeechRecognizerImpl canceling recognition. ";
 
   // The recognition engine is initialized only after STATE_STARTING.
   if (state_ > STATE_STARTING) {
     DCHECK(recognition_engine_.get() != NULL);
     recognition_engine_->EndRecognition();
   }
 
@@ skipping 85 matching lines @@
 
 SpeechRecognizerImpl::FSMState
 SpeechRecognizerImpl::NotFeasible(const FSMEventArgs& event_args) {
   NOTREACHED() << "Unfeasible event " << event_args.event
                << " in state " << state_;
   return state_;
 }
 
 void SpeechRecognizerImpl::CloseAudioControllerAsynchronously() {
   DCHECK(IsCapturingAudio());
+  DCHECK(!using_audio_track_);
   DVLOG(1) << "SpeechRecognizerImpl closing audio controller.";
   // Issues a Close on the audio controller, passing an empty callback. The only
   // purpose of such callback is to keep the audio controller refcounted until
   // Close has completed (in the audio thread) and automatically destroy it
   // afterwards (upon return from OnAudioClosed).
+
   audio_controller_->Close(base::Bind(&SpeechRecognizerImpl::OnAudioClosed,
                                       this, audio_controller_));
   audio_controller_ = NULL;  // The controller is still refcounted by Bind.
   audio_log_->OnClosed(0);
 }
 
 int SpeechRecognizerImpl::GetElapsedTimeMs() const {
   return (num_samples_recorded_ * 1000) / kAudioSampleRate;
 }
 
@@ skipping 27 matching lines @@
 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(FSMEvent event_value)
     : event(event_value),
       audio_data(NULL),
       engine_error(SPEECH_RECOGNITION_ERROR_NONE) {
 }
 
 SpeechRecognizerImpl::FSMEventArgs::~FSMEventArgs() {
 }
 
 }  // namespace content