OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/browser/speech/speech_recognizer.h" | 5 #include "content/browser/speech/speech_recognizer.h" |
6 | 6 |
7 #include "base/basictypes.h" | 7 #include "base/basictypes.h" |
8 #include "base/bind.h" | 8 #include "base/bind.h" |
9 #include "base/time.h" | 9 #include "base/time.h" |
10 #include "content/browser/browser_main_loop.h" | 10 #include "content/browser/browser_main_loop.h" |
11 #include "content/browser/speech/audio_buffer.h" | 11 #include "content/browser/speech/audio_buffer.h" |
12 #include "content/browser/speech/google_one_shot_remote_engine.h" | 12 #include "content/browser/speech/google_one_shot_remote_engine.h" |
13 #include "content/public/browser/browser_thread.h" | 13 #include "content/public/browser/browser_thread.h" |
14 #include "content/public/browser/speech_recognition_event_listener.h" | 14 #include "content/public/browser/speech_recognition_event_listener.h" |
15 #include "content/public/common/speech_recognition_error.h" | 15 #include "content/public/common/speech_recognition_error.h" |
16 #include "content/public/common/speech_recognition_grammar.h" | 16 #include "content/public/common/speech_recognition_grammar.h" |
17 #include "content/public/common/speech_recognition_result.h" | 17 #include "content/public/common/speech_recognition_result.h" |
18 #include "net/url_request/url_request_context_getter.h" | 18 #include "net/url_request/url_request_context_getter.h" |
19 | 19 |
20 using content::BrowserMainLoop; | |
21 using content::BrowserThread; | |
22 using content::SpeechRecognitionError; | |
23 using content::SpeechRecognitionEventListener; | |
24 using content::SpeechRecognitionGrammar; | |
25 using content::SpeechRecognitionResult; | |
26 using media::AudioInputController; | 20 using media::AudioInputController; |
27 using media::AudioManager; | 21 using media::AudioManager; |
28 using media::AudioParameters; | 22 using media::AudioParameters; |
29 using media::ChannelLayout; | 23 using media::ChannelLayout; |
30 | 24 |
| 25 namespace content { |
31 namespace { | 26 namespace { |
32 | 27 |
33 // The following constants are related to the volume level indicator shown in | 28 // The following constants are related to the volume level indicator shown in |
34 // the UI for recorded audio. | 29 // the UI for recorded audio. |
35 // Multiplier used when new volume is greater than previous level. | 30 // Multiplier used when new volume is greater than previous level. |
36 const float kUpSmoothingFactor = 1.0f; | 31 const float kUpSmoothingFactor = 1.0f; |
37 // Multiplier used when new volume is lesser than previous level. | 32 // Multiplier used when new volume is lesser than previous level. |
38 const float kDownSmoothingFactor = 0.7f; | 33 const float kDownSmoothingFactor = 0.7f; |
39 // RMS dB value of a maximum (unclipped) sine wave for int16 samples. | 34 // RMS dB value of a maximum (unclipped) sine wave for int16 samples. |
40 const float kAudioMeterMaxDb = 90.31f; | 35 const float kAudioMeterMaxDb = 90.31f; |
41 // This value corresponds to RMS dB for int16 with 6 most-significant-bits = 0. | 36 // This value corresponds to RMS dB for int16 with 6 most-significant-bits = 0. |
42 // Values lower than this will display as empty level-meter. | 37 // Values lower than this will display as empty level-meter. |
43 const float kAudioMeterMinDb = 30.0f; | 38 const float kAudioMeterMinDb = 30.0f; |
44 const float kAudioMeterDbRange = kAudioMeterMaxDb - kAudioMeterMinDb; | 39 const float kAudioMeterDbRange = kAudioMeterMaxDb - kAudioMeterMinDb; |
45 | 40 |
46 // Maximum level to draw to display unclipped meter. (1.0f displays clipping.) | 41 // Maximum level to draw to display unclipped meter. (1.0f displays clipping.) |
47 const float kAudioMeterRangeMaxUnclipped = 47.0f / 48.0f; | 42 const float kAudioMeterRangeMaxUnclipped = 47.0f / 48.0f; |
48 | 43 |
49 // Returns true if more than 5% of the samples are at min or max value. | 44 // Returns true if more than 5% of the samples are at min or max value. |
50 bool DetectClipping(const speech::AudioChunk& chunk) { | 45 bool DetectClipping(const AudioChunk& chunk) { |
51 const int num_samples = chunk.NumSamples(); | 46 const int num_samples = chunk.NumSamples(); |
52 const int16* samples = chunk.SamplesData16(); | 47 const int16* samples = chunk.SamplesData16(); |
53 const int kThreshold = num_samples / 20; | 48 const int kThreshold = num_samples / 20; |
54 int clipping_samples = 0; | 49 int clipping_samples = 0; |
55 | 50 |
56 for (int i = 0; i < num_samples; ++i) { | 51 for (int i = 0; i < num_samples; ++i) { |
57 if (samples[i] <= -32767 || samples[i] >= 32767) { | 52 if (samples[i] <= -32767 || samples[i] >= 32767) { |
58 if (++clipping_samples > kThreshold) | 53 if (++clipping_samples > kThreshold) |
59 return true; | 54 return true; |
60 } | 55 } |
61 } | 56 } |
62 return false; | 57 return false; |
63 } | 58 } |
64 | 59 |
65 void KeepAudioControllerRefcountedForDtor(scoped_refptr<AudioInputController>) { | 60 void KeepAudioControllerRefcountedForDtor(scoped_refptr<AudioInputController>) { |
66 } | 61 } |
67 | 62 |
68 } // namespace | 63 } // namespace |
69 | 64 |
70 namespace speech { | |
71 | |
72 const int SpeechRecognizer::kAudioSampleRate = 16000; | 65 const int SpeechRecognizer::kAudioSampleRate = 16000; |
73 const ChannelLayout SpeechRecognizer::kChannelLayout = | 66 const ChannelLayout SpeechRecognizer::kChannelLayout = |
74 media::CHANNEL_LAYOUT_MONO; | 67 media::CHANNEL_LAYOUT_MONO; |
75 const int SpeechRecognizer::kNumBitsPerAudioSample = 16; | 68 const int SpeechRecognizer::kNumBitsPerAudioSample = 16; |
76 const int SpeechRecognizer::kNoSpeechTimeoutMs = 8000; | 69 const int SpeechRecognizer::kNoSpeechTimeoutMs = 8000; |
77 const int SpeechRecognizer::kEndpointerEstimationTimeMs = 300; | 70 const int SpeechRecognizer::kEndpointerEstimationTimeMs = 300; |
78 media::AudioManager* SpeechRecognizer::audio_manager_for_tests_ = NULL; | 71 media::AudioManager* SpeechRecognizer::audio_manager_for_tests_ = NULL; |
79 | 72 |
80 COMPILE_ASSERT(SpeechRecognizer::kNumBitsPerAudioSample % 8 == 0, | 73 COMPILE_ASSERT(SpeechRecognizer::kNumBitsPerAudioSample % 8 == 0, |
81 kNumBitsPerAudioSample_must_be_a_multiple_of_8); | 74 kNumBitsPerAudioSample_must_be_a_multiple_of_8); |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
187 event_args.audio_data = new AudioChunk(data, static_cast<size_t>(size), | 180 event_args.audio_data = new AudioChunk(data, static_cast<size_t>(size), |
188 kNumBitsPerAudioSample / 8); | 181 kNumBitsPerAudioSample / 8); |
189 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, | 182 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, |
190 base::Bind(&SpeechRecognizer::DispatchEvent, | 183 base::Bind(&SpeechRecognizer::DispatchEvent, |
191 this, event_args)); | 184 this, event_args)); |
192 } | 185 } |
193 | 186 |
194 void SpeechRecognizer::OnAudioClosed(AudioInputController*) {} | 187 void SpeechRecognizer::OnAudioClosed(AudioInputController*) {} |
195 | 188 |
196 void SpeechRecognizer::OnSpeechRecognitionEngineResult( | 189 void SpeechRecognizer::OnSpeechRecognitionEngineResult( |
197 const content::SpeechRecognitionResult& result) { | 190 const SpeechRecognitionResult& result) { |
198 FSMEventArgs event_args(EVENT_ENGINE_RESULT); | 191 FSMEventArgs event_args(EVENT_ENGINE_RESULT); |
199 event_args.engine_result = result; | 192 event_args.engine_result = result; |
200 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, | 193 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, |
201 base::Bind(&SpeechRecognizer::DispatchEvent, | 194 base::Bind(&SpeechRecognizer::DispatchEvent, |
202 this, event_args)); | 195 this, event_args)); |
203 } | 196 } |
204 | 197 |
205 void SpeechRecognizer::OnSpeechRecognitionEngineError( | 198 void SpeechRecognizer::OnSpeechRecognitionEngineError( |
206 const content::SpeechRecognitionError& error) { | 199 const SpeechRecognitionError& error) { |
207 FSMEventArgs event_args(EVENT_ENGINE_ERROR); | 200 FSMEventArgs event_args(EVENT_ENGINE_ERROR); |
208 event_args.engine_error = error; | 201 event_args.engine_error = error; |
209 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, | 202 BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, |
210 base::Bind(&SpeechRecognizer::DispatchEvent, | 203 base::Bind(&SpeechRecognizer::DispatchEvent, |
211 this, event_args)); | 204 this, event_args)); |
212 } | 205 } |
213 | 206 |
214 // ----------------------- Core FSM implementation --------------------------- | 207 // ----------------------- Core FSM implementation --------------------------- |
215 // TODO(primiano): After the changes in the media package (r129173), this class | 208 // TODO(primiano): After the changes in the media package (r129173), this class |
216 // slightly violates the SpeechRecognitionEventListener interface contract. In | 209 // slightly violates the SpeechRecognitionEventListener interface contract. In |
(...skipping 182 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
399 audio_manager_for_tests_ : | 392 audio_manager_for_tests_ : |
400 BrowserMainLoop::GetAudioManager(); | 393 BrowserMainLoop::GetAudioManager(); |
401 DCHECK(audio_manager != NULL); | 394 DCHECK(audio_manager != NULL); |
402 | 395 |
403 DVLOG(1) << "SpeechRecognizer starting audio capture."; | 396 DVLOG(1) << "SpeechRecognizer starting audio capture."; |
404 num_samples_recorded_ = 0; | 397 num_samples_recorded_ = 0; |
405 audio_level_ = 0; | 398 audio_level_ = 0; |
406 listener_->OnRecognitionStart(session_id_); | 399 listener_->OnRecognitionStart(session_id_); |
407 | 400 |
408 if (!audio_manager->HasAudioInputDevices()) { | 401 if (!audio_manager->HasAudioInputDevices()) { |
409 return Abort(SpeechRecognitionError( | 402 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO, |
410 content::SPEECH_RECOGNITION_ERROR_AUDIO, | 403 SPEECH_AUDIO_ERROR_DETAILS_NO_MIC)); |
411 content::SPEECH_AUDIO_ERROR_DETAILS_NO_MIC)); | |
412 } | 404 } |
413 | 405 |
414 if (audio_manager->IsRecordingInProcess()) { | 406 if (audio_manager->IsRecordingInProcess()) { |
415 return Abort(SpeechRecognitionError( | 407 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO, |
416 content::SPEECH_RECOGNITION_ERROR_AUDIO, | 408 SPEECH_AUDIO_ERROR_DETAILS_IN_USE)); |
417 content::SPEECH_AUDIO_ERROR_DETAILS_IN_USE)); | |
418 } | 409 } |
419 | 410 |
420 const int samples_per_packet = (kAudioSampleRate * | 411 const int samples_per_packet = (kAudioSampleRate * |
421 recognition_engine_->GetDesiredAudioChunkDurationMs()) / 1000; | 412 recognition_engine_->GetDesiredAudioChunkDurationMs()) / 1000; |
422 AudioParameters params(AudioParameters::AUDIO_PCM_LINEAR, kChannelLayout, | 413 AudioParameters params(AudioParameters::AUDIO_PCM_LINEAR, kChannelLayout, |
423 kAudioSampleRate, kNumBitsPerAudioSample, | 414 kAudioSampleRate, kNumBitsPerAudioSample, |
424 samples_per_packet); | 415 samples_per_packet); |
425 audio_controller_ = AudioInputController::Create(audio_manager, this, params); | 416 audio_controller_ = AudioInputController::Create(audio_manager, this, params); |
426 | 417 |
427 if (audio_controller_.get() == NULL) { | 418 if (audio_controller_.get() == NULL) { |
428 return Abort( | 419 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO)); |
429 SpeechRecognitionError(content::SPEECH_RECOGNITION_ERROR_AUDIO)); | |
430 } | 420 } |
431 | 421 |
432 // The endpointer needs to estimate the environment/background noise before | 422 // The endpointer needs to estimate the environment/background noise before |
433 // starting to treat the audio as user input. We wait in the state | 423 // starting to treat the audio as user input. We wait in the state |
434 // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching | 424 // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching |
435 // to user input mode. | 425 // to user input mode. |
436 endpointer_.SetEnvironmentEstimationMode(); | 426 endpointer_.SetEnvironmentEstimationMode(); |
437 audio_controller_->Record(); | 427 audio_controller_->Record(); |
438 return STATE_STARTING; | 428 return STATE_STARTING; |
439 } | 429 } |
(...skipping 24 matching lines...) Expand all Loading... |
464 return STATE_ESTIMATING_ENVIRONMENT; | 454 return STATE_ESTIMATING_ENVIRONMENT; |
465 } | 455 } |
466 } | 456 } |
467 | 457 |
468 SpeechRecognizer::FSMState | 458 SpeechRecognizer::FSMState |
469 SpeechRecognizer::DetectUserSpeechOrTimeout(const FSMEventArgs&) { | 459 SpeechRecognizer::DetectUserSpeechOrTimeout(const FSMEventArgs&) { |
470 if (endpointer_.DidStartReceivingSpeech()) { | 460 if (endpointer_.DidStartReceivingSpeech()) { |
471 listener_->OnSoundStart(session_id_); | 461 listener_->OnSoundStart(session_id_); |
472 return STATE_RECOGNIZING; | 462 return STATE_RECOGNIZING; |
473 } else if (GetElapsedTimeMs() >= kNoSpeechTimeoutMs) { | 463 } else if (GetElapsedTimeMs() >= kNoSpeechTimeoutMs) { |
474 return Abort( | 464 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NO_SPEECH)); |
475 SpeechRecognitionError(content::SPEECH_RECOGNITION_ERROR_NO_SPEECH)); | |
476 } | 465 } |
477 return STATE_WAITING_FOR_SPEECH; | 466 return STATE_WAITING_FOR_SPEECH; |
478 } | 467 } |
479 | 468 |
480 SpeechRecognizer::FSMState | 469 SpeechRecognizer::FSMState |
481 SpeechRecognizer::DetectEndOfSpeech(const FSMEventArgs& event_args) { | 470 SpeechRecognizer::DetectEndOfSpeech(const FSMEventArgs& event_args) { |
482 if (endpointer_.speech_input_complete()) | 471 if (endpointer_.speech_input_complete()) |
483 return StopCaptureAndWaitForResult(event_args); | 472 return StopCaptureAndWaitForResult(event_args); |
484 return STATE_RECOGNIZING; | 473 return STATE_RECOGNIZING; |
485 } | 474 } |
(...skipping 10 matching lines...) Expand all Loading... |
496 listener_->OnSoundEnd(session_id_); | 485 listener_->OnSoundEnd(session_id_); |
497 | 486 |
498 listener_->OnAudioEnd(session_id_); | 487 listener_->OnAudioEnd(session_id_); |
499 return STATE_WAITING_FINAL_RESULT; | 488 return STATE_WAITING_FINAL_RESULT; |
500 } | 489 } |
501 | 490 |
502 SpeechRecognizer::FSMState | 491 SpeechRecognizer::FSMState |
503 SpeechRecognizer::AbortSilently(const FSMEventArgs& event_args) { | 492 SpeechRecognizer::AbortSilently(const FSMEventArgs& event_args) { |
504 DCHECK_NE(event_args.event, EVENT_AUDIO_ERROR); | 493 DCHECK_NE(event_args.event, EVENT_AUDIO_ERROR); |
505 DCHECK_NE(event_args.event, EVENT_ENGINE_ERROR); | 494 DCHECK_NE(event_args.event, EVENT_ENGINE_ERROR); |
506 return Abort( | 495 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NONE)); |
507 SpeechRecognitionError(content::SPEECH_RECOGNITION_ERROR_NONE)); | |
508 } | 496 } |
509 | 497 |
510 SpeechRecognizer::FSMState | 498 SpeechRecognizer::FSMState |
511 SpeechRecognizer::AbortWithError(const FSMEventArgs& event_args) { | 499 SpeechRecognizer::AbortWithError(const FSMEventArgs& event_args) { |
512 if (event_args.event == EVENT_AUDIO_ERROR) { | 500 if (event_args.event == EVENT_AUDIO_ERROR) { |
513 return Abort( | 501 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO)); |
514 SpeechRecognitionError(content::SPEECH_RECOGNITION_ERROR_AUDIO)); | |
515 } else if (event_args.event == EVENT_ENGINE_ERROR) { | 502 } else if (event_args.event == EVENT_ENGINE_ERROR) { |
516 return Abort(event_args.engine_error); | 503 return Abort(event_args.engine_error); |
517 } | 504 } |
518 return Abort( | 505 return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_ABORTED)); |
519 SpeechRecognitionError(content::SPEECH_RECOGNITION_ERROR_ABORTED)); | |
520 } | 506 } |
521 | 507 |
522 SpeechRecognizer::FSMState SpeechRecognizer::Abort( | 508 SpeechRecognizer::FSMState SpeechRecognizer::Abort( |
523 const SpeechRecognitionError& error) { | 509 const SpeechRecognitionError& error) { |
524 if (IsCapturingAudio()) | 510 if (IsCapturingAudio()) |
525 CloseAudioControllerAsynchronously(); | 511 CloseAudioControllerAsynchronously(); |
526 | 512 |
527 DVLOG(1) << "SpeechRecognizer canceling recognition. "; | 513 DVLOG(1) << "SpeechRecognizer canceling recognition. "; |
528 | 514 |
529 // The recognition engine is initialized only after STATE_STARTING. | 515 // The recognition engine is initialized only after STATE_STARTING. |
530 if (state_ > STATE_STARTING) { | 516 if (state_ > STATE_STARTING) { |
531 DCHECK(recognition_engine_.get() != NULL); | 517 DCHECK(recognition_engine_.get() != NULL); |
532 recognition_engine_->EndRecognition(); | 518 recognition_engine_->EndRecognition(); |
533 } | 519 } |
534 | 520 |
535 if (state_ > STATE_WAITING_FOR_SPEECH && state_ < STATE_WAITING_FINAL_RESULT) | 521 if (state_ > STATE_WAITING_FOR_SPEECH && state_ < STATE_WAITING_FINAL_RESULT) |
536 listener_->OnSoundEnd(session_id_); | 522 listener_->OnSoundEnd(session_id_); |
537 | 523 |
538 if (state_ > STATE_STARTING && state_ < STATE_WAITING_FINAL_RESULT) | 524 if (state_ > STATE_STARTING && state_ < STATE_WAITING_FINAL_RESULT) |
539 listener_->OnAudioEnd(session_id_); | 525 listener_->OnAudioEnd(session_id_); |
540 | 526 |
541 if (error.code != content::SPEECH_RECOGNITION_ERROR_NONE) | 527 if (error.code != SPEECH_RECOGNITION_ERROR_NONE) |
542 listener_->OnRecognitionError(session_id_, error); | 528 listener_->OnRecognitionError(session_id_, error); |
543 | 529 |
544 listener_->OnRecognitionEnd(session_id_); | 530 listener_->OnRecognitionEnd(session_id_); |
545 | 531 |
546 return STATE_IDLE; | 532 return STATE_IDLE; |
547 } | 533 } |
548 | 534 |
549 SpeechRecognizer::FSMState SpeechRecognizer::ProcessIntermediateResult( | 535 SpeechRecognizer::FSMState SpeechRecognizer::ProcessIntermediateResult( |
550 const FSMEventArgs& event_args) { | 536 const FSMEventArgs& event_args) { |
551 // Provisional results can occur only during continuous (non one-shot) mode. | 537 // Provisional results can occur only during continuous (non one-shot) mode. |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
653 | 639 |
654 void SpeechRecognizer::SetAudioManagerForTests( | 640 void SpeechRecognizer::SetAudioManagerForTests( |
655 AudioManager* audio_manager) { | 641 AudioManager* audio_manager) { |
656 audio_manager_for_tests_ = audio_manager; | 642 audio_manager_for_tests_ = audio_manager; |
657 } | 643 } |
658 | 644 |
659 SpeechRecognizer::FSMEventArgs::FSMEventArgs(FSMEvent event_value) | 645 SpeechRecognizer::FSMEventArgs::FSMEventArgs(FSMEvent event_value) |
660 : event(event_value), | 646 : event(event_value), |
661 audio_error_code(0), | 647 audio_error_code(0), |
662 audio_data(NULL), | 648 audio_data(NULL), |
663 engine_error(content::SPEECH_RECOGNITION_ERROR_NONE) { | 649 engine_error(SPEECH_RECOGNITION_ERROR_NONE) { |
664 } | 650 } |
665 | 651 |
666 SpeechRecognizer::FSMEventArgs::~FSMEventArgs() { | 652 SpeechRecognizer::FSMEventArgs::~FSMEventArgs() { |
667 } | 653 } |
668 | 654 |
669 } // namespace speech | 655 } // namespace content |
OLD | NEW |