Binding media stream audio track to speech recognition [renderer]

content/renderer/speech_recognition_dispatcher.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/speech_recognition_dispatcher.h"
 
 #include "base/basictypes.h"
 #include "base/strings/utf_string_conversions.h"
 #include "content/common/speech_recognition_messages.h"
+#include "content/renderer/media/media_stream_audio_source.h"
 #include "content/renderer/render_view_impl.h"
+#include "content/renderer/speech_recognition_audio_source_provider.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 #include "third_party/WebKit/public/platform/WebVector.h"
 #include "third_party/WebKit/public/web/WebSpeechGrammar.h"
 #include "third_party/WebKit/public/web/WebSpeechRecognitionParams.h"
 #include "third_party/WebKit/public/web/WebSpeechRecognitionResult.h"
 #include "third_party/WebKit/public/web/WebSpeechRecognizerClient.h"
 
 using blink::WebVector;
 using blink::WebString;
 using blink::WebSpeechGrammar;
 using blink::WebSpeechRecognitionHandle;
 using blink::WebSpeechRecognitionResult;
 using blink::WebSpeechRecognitionParams;
 using blink::WebSpeechRecognizerClient;
 
 namespace content {
 
 SpeechRecognitionDispatcher::SpeechRecognitionDispatcher(
     RenderViewImpl* render_view)
     : RenderViewObserver(render_view),
       recognizer_client_(NULL),
+      audio_track_set_(false),
+      is_allowed_audio_track_(false),
+      render_loop_(base::MessageLoopProxy::current()),
       next_id_(1) {
 }
 
 SpeechRecognitionDispatcher::~SpeechRecognitionDispatcher() {
 }
 
 void SpeechRecognitionDispatcher::AbortAllRecognitions() {
+  audio_source_provider_.reset();
   Send(new SpeechRecognitionHostMsg_AbortAllRequests(
       routing_id()));
 }
 
 bool SpeechRecognitionDispatcher::OnMessageReceived(
     const IPC::Message& message) {
   bool handled = true;
   IPC_BEGIN_MESSAGE_MAP(SpeechRecognitionDispatcher, message)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_Started, OnRecognitionStarted)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_AudioStarted, OnAudioStarted)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_SoundStarted, OnSoundStarted)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_SoundEnded, OnSoundEnded)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_AudioEnded, OnAudioEnded)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_ErrorOccurred, OnErrorOccurred)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_Ended, OnRecognitionEnded)
     IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_ResultRetrieved,
                         OnResultsRetrieved)
+    IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_AudioTrackReady,
+                        OnAudioTrackReady)
+    IPC_MESSAGE_HANDLER(SpeechRecognitionMsg_AudioChunkProcessed,
+                        OnAudioChunkProcessed)
     IPC_MESSAGE_UNHANDLED(handled = false)
   IPC_END_MESSAGE_MAP()
   return handled;
 }
 
+void SpeechRecognitionDispatcher::attach(
+    const blink::WebSpeechRecognitionHandle& handle,
+    const blink::WebMediaStreamTrack& audio_track,
+    blink::WebSpeechRecognizerClient* recognizer_client) {
+
+  // Check if track is from an allowed source (microphone only for now)
+  // TODO(burnik): externalize the policy of allowed track types from dispatcher
+  DCHECK(audio_track.source().type() == blink::WebMediaStreamSource::TypeAudio);
+  MediaStreamAudioSource* native_source =
+      static_cast <MediaStreamAudioSource*>(audio_track.source().extraData());
+  StreamDeviceInfo device_info = native_source->device_info();
+  is_allowed_audio_track_ = (device_info.device.type ==
+    content::MEDIA_DEVICE_AUDIO_CAPTURE);
+
+  audio_track_ = audio_track;
+  audio_track_set_ = true;
+}
+
+void SpeechRecognitionDispatcher::detach(
+    const blink::WebSpeechRecognitionHandle& handle,
+    blink::WebSpeechRecognizerClient* recognizer_client) {
+  audio_track_set_ = false;
+}
+
 void SpeechRecognitionDispatcher::start(
     const WebSpeechRecognitionHandle& handle,
     const WebSpeechRecognitionParams& params,
     WebSpeechRecognizerClient* recognizer_client) {
   DCHECK(!recognizer_client_ || recognizer_client_ == recognizer_client);
   recognizer_client_ = recognizer_client;
 
+  // destroy any previous instance not to starve it waiting on chunk ACKs
+  audio_source_provider_.reset();
+
+  if (audio_track_set_ && !is_allowed_audio_track_) {
+    // notify user that the track used is not supported
+    recognizer_client_->didReceiveError(
+          handle,
+          WebString("Provided audioTrack is not supported. Ignoring track."),
+          WebSpeechRecognizerClient::NotAllowedError);
+  }
+
   SpeechRecognitionHostMsg_StartRequest_Params msg_params;
   for (size_t i = 0; i < params.grammars().size(); ++i) {
     const WebSpeechGrammar& grammar = params.grammars()[i];
     msg_params.grammars.push_back(
         SpeechRecognitionGrammar(grammar.src().spec(), grammar.weight()));
   }
   msg_params.language = base::UTF16ToUTF8(params.language());
   msg_params.max_hypotheses = static_cast<uint32>(params.maxAlternatives());
   msg_params.continuous = params.continuous();
   msg_params.interim_results = params.interimResults();
   msg_params.origin_url = params.origin().toString().utf8();
   msg_params.render_view_id = routing_id();
   msg_params.request_id = GetOrCreateIDForHandle(handle);
+  // fall back to default input when the track is not allowed
+  msg_params.using_audio_track = (audio_track_set_ && is_allowed_audio_track_);
   // The handle mapping will be removed in |OnRecognitionEnd|.
   Send(new SpeechRecognitionHostMsg_StartRequest(msg_params));
 }
 
 void SpeechRecognitionDispatcher::stop(
     const WebSpeechRecognitionHandle& handle,
     WebSpeechRecognizerClient* recognizer_client) {
+  audio_source_provider_.reset();
   // Ignore a |stop| issued without a matching |start|.
   if (recognizer_client_ != recognizer_client || !HandleExists(handle))
     return;
   Send(new SpeechRecognitionHostMsg_StopCaptureRequest(
       routing_id(), GetOrCreateIDForHandle(handle)));
 }
 
 void SpeechRecognitionDispatcher::abort(
     const WebSpeechRecognitionHandle& handle,
     WebSpeechRecognizerClient* recognizer_client) {
+  audio_source_provider_.reset();
   // Ignore an |abort| issued without a matching |start|.
   if (recognizer_client_ != recognizer_client || !HandleExists(handle))
     return;
   Send(new SpeechRecognitionHostMsg_AbortRequest(
       routing_id(), GetOrCreateIDForHandle(handle)));
 }
 
 void SpeechRecognitionDispatcher::OnRecognitionStarted(int request_id) {
   recognizer_client_->didStart(GetHandleFromID(request_id));
 }
@@ skipping 39 matching lines @@
   NOTREACHED();
   return WebSpeechRecognizerClient::OtherError;
 }
 
 void SpeechRecognitionDispatcher::OnErrorOccurred(
     int request_id, const SpeechRecognitionError& error) {
   if (error.code == SPEECH_RECOGNITION_ERROR_NO_MATCH) {
     recognizer_client_->didReceiveNoMatch(GetHandleFromID(request_id),
                                           WebSpeechRecognitionResult());
   } else {
+    audio_source_provider_.reset();
     recognizer_client_->didReceiveError(
         GetHandleFromID(request_id),
         WebString(),  // TODO(primiano): message?
         WebKitErrorCode(error.code));
   }
 }
 
 void SpeechRecognitionDispatcher::OnRecognitionEnded(int request_id) {
   // TODO(tommi): It is possible that the handle isn't found in the array if
   // the user just refreshed the page. It seems that we then get a notification
   // for the previously loaded instance of the page.
   HandleMap::iterator iter = handle_map_.find(request_id);
   if (iter == handle_map_.end()) {
     DLOG(ERROR) << "OnRecognitionEnded called for a handle that doesn't exist";
   } else {
     WebSpeechRecognitionHandle handle = iter->second;
     // Note: we need to erase the handle from the map *before* calling didEnd.
     // didEnd may call back synchronously to start a new recognition session,
     // and we don't want to delete the handle from the map after that happens.
     handle_map_.erase(request_id);
+    audio_source_provider_.reset();
     recognizer_client_->didEnd(handle);
   }
 }
 
 void SpeechRecognitionDispatcher::OnResultsRetrieved(
     int request_id, const SpeechRecognitionResults& results) {
   size_t provisional_count = 0;
   SpeechRecognitionResults::const_iterator it = results.begin();
   for (; it != results.end(); ++it) {
     if (it->is_provisional)
@@ skipping 17 matching lines @@
       transcripts[i] = result.hypotheses[i].utterance;
       confidences[i] = static_cast<float>(result.hypotheses[i].confidence);
     }
     webkit_result->assign(transcripts, confidences, !result.is_provisional);
   }
 
   recognizer_client_->didReceiveResults(
       GetHandleFromID(request_id), final, provisional);
 }
 
+void SpeechRecognitionDispatcher::OnAudioError(int request_id){
+  // Browser gets notified on render thread
+  if (!render_loop_->BelongsToCurrentThread()) {
+    render_loop_->PostTask(
+        FROM_HERE,
+        base::Bind(&SpeechRecognitionDispatcher::OnAudioError,
+                   base::Unretained(this), request_id));
+    return;
+  }
+  audio_source_provider_.reset();
+}
+
+void SpeechRecognitionDispatcher::OnAudioTrackReady(
+    int request_id,
+    const media::AudioParameters& params,
+    base::SharedMemoryHandle handle,
+    uint32 length) {
+
+  // TODO(burnik): Log and DCHECK(!audio_source_provider_).
+  if (audio_track_.isNull()) {
+    audio_source_provider_.reset();
+    return;
+  }
+
+  audio_source_provider_.reset(
+      new SpeechRecognitionAudioSourceProvider(
+          audio_track_, params, handle, length,
+          base::Bind(&SpeechRecognitionDispatcher::OnAudioData,
+                     base::Unretained(this), request_id),
+          base::Bind(&SpeechRecognitionDispatcher::OnAudioError,
+                     base::Unretained(this), request_id)));
+}
+
+void SpeechRecognitionDispatcher::OnAudioChunkProcessed(
+    int request_id) {
+
+  // TODO(burnik): Log and DCHECK(!audio_source_provider_).
+  if (audio_track_.isNull())
+    return;
+
+  // discard any message to a destroyed instance
+  if(!audio_source_provider_.get())
+      return;
+
+  audio_source_provider_->NotifyAudioBusConsumed();
+}
+
+
+// TODO(burnik): Consider using sync_socket
+void SpeechRecognitionDispatcher::OnAudioData(int request_id) {
+  // Browser gets notified on render thread
+  if (!render_loop_->BelongsToCurrentThread()) {
+    render_loop_->PostTask(
+        FROM_HERE,
+        base::Bind(&SpeechRecognitionDispatcher::OnAudioData,
+                   base::Unretained(this), request_id));
+    return;
+  }
+  // If the handle isn't found in the array, which might happen if the
+  // recognition has been ended by the browser, delete the
+  // |audio_source_provider_|.
+  HandleMap::iterator iter = handle_map_.find(request_id);
+  if (iter == handle_map_.end()) {
+    audio_source_provider_.reset();
+    return;
+  }
+
+  Send(new SpeechRecognitionHostMsg_OnAudioTrackData(routing_id(), request_id));
+}
 
 int SpeechRecognitionDispatcher::GetOrCreateIDForHandle(
     const WebSpeechRecognitionHandle& handle) {
   // Search first for an existing mapping.
   for (HandleMap::iterator iter = handle_map_.begin();
       iter != handle_map_.end();
       ++iter) {
     if (iter->second.equals(handle))
       return iter->first;
   }
@@ skipping 16 matching lines @@
 }
 
 const WebSpeechRecognitionHandle& SpeechRecognitionDispatcher::GetHandleFromID(
     int request_id) {
   HandleMap::iterator iter = handle_map_.find(request_id);
   DCHECK(iter != handle_map_.end());
   return iter->second;
 }
 
 }  // namespace content