Add an endpointer for detecting end of speech.

chrome/browser/speech/speech_recognizer.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/speech/speech_recognizer.h"
 
 #include "base/ref_counted.h"
 #include "base/scoped_ptr.h"
+#include "base/time.h"
 #include "chrome/browser/chrome_thread.h"
 #include "chrome/browser/profile.h"
 #include "chrome/common/net/url_request_context_getter.h"
 #include "third_party/speex/include/speex/speex.h"
 
 using media::AudioInputController;
 using std::list;
 using std::string;
 
 namespace {
@@ skipping 17 matching lines @@
 namespace speech_input {
 
 // Provides a simple interface to encode raw audio using the Speex codec.
 class SpeexEncoder {
  public:
   SpeexEncoder();
   ~SpeexEncoder();
 
   int samples_per_frame() const { return samples_per_frame_; }
 
-  // Encodes each frame of raw audio in |raw_samples| and adds the
+  // Encodes each frame of raw audio in |samples| and adds the
   // encoded frames as a set of strings to the |encoded_frames| list.
   // Ownership of the newly added strings is transferred to the caller.
-  void Encode(const string& raw_samples,
+  void Encode(const short* samples,
+              int num_samples,
               std::list<std::string*>* encoded_frames);
 
  private:
   SpeexBits bits_;
   void* encoder_state_;
   int samples_per_frame_;
   char encoded_frame_data_[kMaxSpeexFrameLength + 1];  // +1 for the frame size.
 };
 
 SpeexEncoder::SpeexEncoder() {
   speex_bits_init(&bits_);
   encoder_state_ = speex_encoder_init(&speex_wb_mode);
   DCHECK(encoder_state_);
   speex_encoder_ctl(encoder_state_, SPEEX_GET_FRAME_SIZE, &samples_per_frame_);
   DCHECK(samples_per_frame_ > 0);
   int quality = kSpeexEncodingQuality;
   speex_encoder_ctl(encoder_state_, SPEEX_SET_QUALITY, &quality);
   int vbr = 1;
   speex_encoder_ctl(encoder_state_, SPEEX_SET_VBR, &vbr);
 }
 
 SpeexEncoder::~SpeexEncoder() {
   speex_bits_destroy(&bits_);
   speex_encoder_destroy(encoder_state_);
 }
 
-void SpeexEncoder::Encode(const string& raw_samples,
+void SpeexEncoder::Encode(const short* samples,
+                          int num_samples,
                           std::list<std::string*>* encoded_frames) {
-  const short* samples = reinterpret_cast<const short*>(raw_samples.data());
-  DCHECK((raw_samples.length() % sizeof(short)) == 0);
-  int num_samples = raw_samples.length() / sizeof(short);
-
   // Drop incomplete frames, typically those which come in when recording stops.
   num_samples -= (num_samples % samples_per_frame_);
   for (int i = 0; i < num_samples; i += samples_per_frame_) {
     speex_bits_reset(&bits_);
     speex_encode_int(encoder_state_, const_cast<spx_int16_t*>(samples + i),
                      &bits_);
 
     // Encode the frame and place the size of the frame as the first byte. This
     // is the packet format for MIME type x-speex-with-header-byte.
     int frame_length = speex_bits_write(&bits_, encoded_frame_data_ + 1,
                                         kMaxSpeexFrameLength);
     encoded_frame_data_[0] = static_cast<char>(frame_length);
     encoded_frames->push_back(new string(encoded_frame_data_,
                                          frame_length + 1));
   }
 }
 
 SpeechRecognizer::SpeechRecognizer(Delegate* delegate,
                                    const SpeechInputCallerId& caller_id)
     : delegate_(delegate),
       caller_id_(caller_id),
-      encoder_(new SpeexEncoder()) {
+      encoder_(new SpeexEncoder()),
+      endpointer_(kAudioSampleRate) {
+  endpointer_.set_speech_input_complete_silence_length(
+      base::Time::kMicrosecondsPerSecond / 2);
+  endpointer_.set_long_speech_input_complete_silence_length(
+      base::Time::kMicrosecondsPerSecond);
+  endpointer_.set_long_speech_length(3 * base::Time::kMicrosecondsPerSecond);
+  endpointer_.StartSession();
 }
 
 SpeechRecognizer::~SpeechRecognizer() {
   // Recording should have stopped earlier due to the endpointer or
   // |StopRecording| being called.
   DCHECK(!audio_controller_.get());
   DCHECK(!request_.get() || !request_->HasPendingRequest());
   DCHECK(audio_buffers_.empty());
+  endpointer_.EndSession();
 }
 
 bool SpeechRecognizer::StartRecording() {
   DCHECK(ChromeThread::CurrentlyOn(ChromeThread::IO));
   DCHECK(!audio_controller_.get());
   DCHECK(!request_.get() || !request_->HasPendingRequest());
 
+  // TODO(satish): Normally for a short time (even 0.5s) the endpointer needs to
+  // estimate the environment/background noise before starting to treat the
+  // audio as user input. Once we have implemented a popup UI to notify the user
+  // that recording has started, we should perhaps have a short interval where
+  // we record background audio and then show the popup UI so that the user can
+  // start speaking after that. For now we just do these together so there isn't
+  // any background noise for the end pointer (still works ok).
+  endpointer_.SetEnvironmentEstimationMode();
+  endpointer_.SetUserInputMode();
+
   int samples_per_packet = (kAudioSampleRate * kAudioPacketIntervalMs) / 1000;
   DCHECK((samples_per_packet % encoder_->samples_per_frame()) == 0);
   audio_controller_ = AudioInputController::Create(this,
       AudioManager::AUDIO_PCM_LINEAR, kNumAudioChannels,
       kAudioSampleRate, kNumBitsPerAudioSample,
       samples_per_packet);
   DCHECK(audio_controller_.get());
   LOG(INFO) << "SpeechRecognizer starting record.";
   audio_controller_->Record();
 
@@ skipping 20 matching lines @@
   DCHECK(ChromeThread::CurrentlyOn(ChromeThread::IO));
 
   // If audio recording has already stopped and we are in recognition phase,
   // silently ignore any more calls to stop recording.
   if (!audio_controller_.get())
     return;
 
   LOG(INFO) << "SpeechRecognizer stopping record.";
   audio_controller_->Close();
   audio_controller_ = NULL;  // Releases the ref ptr.
+
   delegate_->DidCompleteRecording(caller_id_);
 
   // If we haven't got any audio yet end the recognition sequence here.
   if (audio_buffers_.empty()) {
     // Guard against the delegate freeing us until we finish our job.
     scoped_refptr<SpeechRecognizer> me(this);
     delegate_->DidCompleteRecognition(caller_id_);
     return;
   }
 
@@ skipping 64 matching lines @@
 
 void SpeechRecognizer::HandleOnData(string* data) {
   // Check if we are still recording and if not discard this buffer, as
   // recording might have been stopped after this buffer was posted to the queue
   // by |OnData|.
   if (!audio_controller_.get()) {
     delete data;
     return;
   }
 
-  encoder_->Encode(*data, &audio_buffers_);
+  const short* samples = reinterpret_cast<const short*>(data->data());
+  DCHECK((data->length() % sizeof(short)) == 0);
+  int num_samples = data->length() / sizeof(short);
+
+  encoder_->Encode(samples, num_samples, &audio_buffers_);
+  endpointer_.ProcessAudio(samples, num_samples);
   delete data;
 
+  if (endpointer_.speech_input_complete()) {
+    StopRecording();
+  }
+
   // TODO(satish): Once we have streaming POST, start sending the data received
   // here as POST chunks.
 }
 
 void SpeechRecognizer::SetRecognitionResult(bool error, const string16& value) {
   delegate_->SetRecognitionResult(caller_id_, error, value);
 
   // Guard against the delegate freeing us until we finish our job.
   scoped_refptr<SpeechRecognizer> me(this);
   delegate_->DidCompleteRecognition(caller_id_);
 }
 
 }  // namespace speech_input