Stream speech audio to server as it gets recorded, instead of waiting until end of recording.

content/browser/speech/speech_recognizer.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/speech/speech_recognizer.h"
 
 #include "base/time.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/common/net/url_request_context_getter.h"
 #include "content/browser/browser_thread.h"
@@ skipping 92 matching lines @@
                                       kNumBitsPerAudioSample));
   int samples_per_packet = (kAudioSampleRate * kAudioPacketIntervalMs) / 1000;
   AudioParameters params(AudioParameters::AUDIO_PCM_LINEAR, kNumAudioChannels,
                          kAudioSampleRate, kNumBitsPerAudioSample,
                          samples_per_packet);
   audio_controller_ = AudioInputController::Create(this, params);
   DCHECK(audio_controller_.get());
   VLOG(1) << "SpeechRecognizer starting record.";
   num_samples_recorded_ = 0;
   audio_controller_->Record();
+  previous_audio_chunk_.clear();
 
   return true;
 }
 
 void SpeechRecognizer::CancelRecognition() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK(audio_controller_.get() || request_.get());
 
   // Stop recording if required.
   if (audio_controller_.get()) {
@@ skipping 11 matching lines @@
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
   // If audio recording has already stopped and we are in recognition phase,
   // silently ignore any more calls to stop recording.
   if (!audio_controller_.get())
     return;
 
   VLOG(1) << "SpeechRecognizer stopping record.";
   audio_controller_->Close();
   audio_controller_ = NULL;  // Releases the ref ptr.
-  encoder_->Flush();
 
   delegate_->DidCompleteRecording(caller_id_);
 
-  // Since the http request takes a single string as POST data, allocate
-  // one and copy over bytes from the audio buffers to the string.
-  // And If we haven't got any audio yet end the recognition sequence here.
-  string mime_type = encoder_->mime_type();
-  string data;
-  encoder_->GetEncodedData(&data);
+  // Get any last bits of encoded data left.
+  encoder_->Flush();
+  string encoded_data;
+  encoder_->GetEncodedDataAndClear(&encoded_data);
   encoder_.reset();
 
-  if (data.empty()) {
+  // If we haven't got any audio yet end the recognition sequence here.
+  if (request_ == NULL) {
     // Guard against the delegate freeing us until we finish our job.
     scoped_refptr<SpeechRecognizer> me(this);
     delegate_->DidCompleteRecognition(caller_id_);
   } else {
-    DCHECK(!request_.get());
-    request_.reset(new SpeechRecognitionRequest(
-        Profile::GetDefaultRequestContext(), this));
-    request_->Send(language_, grammar_, hardware_info_, origin_url_,
-                   mime_type, data);
+    // UploadAudioChunk requires a non-empty buffer. So we check if there was
+    // any data available since last time we sent and if not try to send the
+    // last chunk again (i.e. repeat the last 100ms of audio). If nothing was
+    // recorded yet, we just send a whitespace string.

[bulach, 2011/03/03 20:05:00]

hmm, I haven't followed the other CL this one depends, but wouldn't it be
simpler to have just a "FinishChunk" (or something similar)? worst case, it may
be simpler to move the "emptiness" test down to UploadAudioChunk rather than
here?

also, why is it necessary to send previoud_audio_chunk again?

if both we can't work around these limitations, please also clarify why repeat
the last chunk.

+    if (encoded_data.empty()) {
+      encoded_data = !previous_audio_chunk_.empty() ?
+                     previous_audio_chunk_ : " ";
+    }
+    request_->UploadAudioChunk(encoded_data, true);
   }
 }
 
-void SpeechRecognizer::ReleaseAudioBuffers() {
-}
-
 // Invoked in the audio thread.
 void SpeechRecognizer::OnError(AudioInputController* controller,
                                int error_code) {
   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
                          NewRunnableMethod(this,
                                            &SpeechRecognizer::HandleOnError,
                                            error_code));
 }
 
 void SpeechRecognizer::HandleOnError(int error_code) {
@@ skipping 25 matching lines @@
   // recording might have been stopped after this buffer was posted to the queue
   // by |OnData|.
   if (!audio_controller_.get()) {
     delete data;
     return;
   }
 
   const short* samples = reinterpret_cast<const short*>(data->data());
   DCHECK((data->length() % sizeof(short)) == 0);
   int num_samples = data->length() / sizeof(short);
-
   encoder_->Encode(samples, num_samples);
   float rms;
   endpointer_.ProcessAudio(samples, num_samples, &rms);
   bool did_clip = Clipping(samples, num_samples);
   delete data;
   num_samples_recorded_ += num_samples;
 
+  if (request_ == NULL) {
+    // This was the first audio packet recorded, so start a request to the
+    // server to send the data.
+    request_.reset(new SpeechRecognitionRequest(
+        Profile::GetDefaultRequestContext(), this));
+    request_->Start(language_, grammar_, hardware_info_, origin_url_,
+                    encoder_->mime_type());
+  }
+
+  string encoded_data;
+  encoder_->GetEncodedDataAndClear(&encoded_data);
+  DCHECK(!encoded_data.empty());
+  request_->UploadAudioChunk(encoded_data, false);
+  previous_audio_chunk_ = encoded_data;
+
   if (endpointer_.IsEstimatingEnvironment()) {
     // Check if we have gathered enough audio for the endpointer to do
     // environment estimation and should move on to detect speech/end of speech.
     if (num_samples_recorded_ >= (kEndpointerEstimationTimeMs *
                                   kAudioSampleRate) / 1000) {
       endpointer_.SetUserInputMode();
       delegate_->DidCompleteEnvironmentEstimation(caller_id_);
     }
     return;  // No more processing since we are still estimating environment.
   }
@@ skipping 48 matching lines @@
 
 void SpeechRecognizer::InformErrorAndCancelRecognition(ErrorCode error) {
   CancelRecognition();
 
   // Guard against the delegate freeing us until we finish our job.
   scoped_refptr<SpeechRecognizer> me(this);
   delegate_->OnRecognizerError(caller_id_, error);
 }
 
 }  // namespace speech_input