| OLD | NEW |
| 1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "content/renderer/media/speech_recognition_audio_sink.h" | 5 #include "content/renderer/media/speech_recognition_audio_sink.h" |
| 6 | 6 |
| 7 #include <stddef.h> | 7 #include <stddef.h> |
| 8 #include <stdint.h> | 8 #include <stdint.h> |
| 9 #include <string.h> | 9 #include <string.h> |
| 10 #include <utility> | 10 #include <utility> |
| 11 | 11 |
| 12 #include "base/bind.h" | 12 #include "base/bind.h" |
| 13 #include "base/macros.h" | 13 #include "base/macros.h" |
| 14 #include "base/strings/utf_string_conversions.h" | |
| 15 #include "content/renderer/media/media_stream_audio_source.h" | 14 #include "content/renderer/media/media_stream_audio_source.h" |
| 16 #include "content/renderer/media/webrtc/webrtc_local_audio_track_adapter.h" | 15 #include "content/renderer/media/media_stream_audio_track.h" |
| 17 #include "content/renderer/media/webrtc_local_audio_track.h" | 16 #include "content/renderer/media/webrtc/mock_peer_connection_dependency_factory.
h" |
| 18 #include "media/base/audio_bus.h" | 17 #include "media/base/audio_bus.h" |
| 19 #include "media/base/audio_parameters.h" | 18 #include "media/base/audio_parameters.h" |
| 20 #include "testing/gmock/include/gmock/gmock.h" | 19 #include "testing/gmock/include/gmock/gmock.h" |
| 21 #include "testing/gtest/include/gtest/gtest.h" | 20 #include "testing/gtest/include/gtest/gtest.h" |
| 22 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" | 21 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" |
| 22 #include "third_party/WebKit/public/platform/WebString.h" |
| 23 #include "third_party/WebKit/public/web/WebHeap.h" | 23 #include "third_party/WebKit/public/web/WebHeap.h" |
| 24 | 24 |
| 25 namespace { | 25 namespace { |
| 26 | 26 |
| 27 // Supported speech recognition audio parameters. | 27 // Supported speech recognition audio parameters. |
| 28 const int kSpeechRecognitionSampleRate = 16000; | 28 const int kSpeechRecognitionSampleRate = 16000; |
| 29 const int kSpeechRecognitionFramesPerBuffer = 1600; | 29 const int kSpeechRecognitionFramesPerBuffer = 1600; |
| 30 | 30 |
| 31 // Input audio format. | 31 // Input audio format. |
| 32 const media::AudioParameters::Format kInputFormat = | 32 const media::AudioParameters::Format kInputFormat = |
| (...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 197 // Audio bus wrapping the shared memory from the renderer. | 197 // Audio bus wrapping the shared memory from the renderer. |
| 198 std::unique_ptr<media::AudioBus> audio_track_bus_; | 198 std::unique_ptr<media::AudioBus> audio_track_bus_; |
| 199 | 199 |
| 200 DISALLOW_COPY_AND_ASSIGN(FakeSpeechRecognizer); | 200 DISALLOW_COPY_AND_ASSIGN(FakeSpeechRecognizer); |
| 201 }; | 201 }; |
| 202 | 202 |
| 203 } // namespace | 203 } // namespace |
| 204 | 204 |
| 205 namespace content { | 205 namespace content { |
| 206 | 206 |
| 207 namespace { |
| 208 |
| 209 class TestDrivenAudioSource : public MediaStreamAudioSource { |
| 210 public: |
| 211 TestDrivenAudioSource() : MediaStreamAudioSource(true) {} |
| 212 ~TestDrivenAudioSource() final {} |
| 213 |
| 214 // Expose protected methods as public for testing. |
| 215 using MediaStreamAudioSource::SetFormat; |
| 216 using MediaStreamAudioSource::DeliverDataToTracks; |
| 217 }; |
| 218 |
| 219 } // namespace |
| 220 |
| 207 class SpeechRecognitionAudioSinkTest : public testing::Test { | 221 class SpeechRecognitionAudioSinkTest : public testing::Test { |
| 208 public: | 222 public: |
| 209 SpeechRecognitionAudioSinkTest() {} | 223 SpeechRecognitionAudioSinkTest() {} |
| 210 | 224 |
| 211 ~SpeechRecognitionAudioSinkTest() { | 225 ~SpeechRecognitionAudioSinkTest() { |
| 226 blink_source_.reset(); |
| 227 blink_track_.reset(); |
| 212 speech_audio_sink_.reset(); | 228 speech_audio_sink_.reset(); |
| 213 blink::WebHeap::collectAllGarbageForTesting(); | 229 blink::WebHeap::collectAllGarbageForTesting(); |
| 214 } | 230 } |
| 215 | 231 |
| 216 // Initializes the producer and consumer with specified audio parameters. | 232 // Initializes the producer and consumer with specified audio parameters. |
| 217 // Returns the minimal number of input audio buffers which need to be captured | 233 // Returns the minimal number of input audio buffers which need to be captured |
| 218 // before they get sent to the consumer. | 234 // before they get sent to the consumer. |
| 219 uint32_t Initialize(int input_sample_rate, | 235 uint32_t Initialize(int input_sample_rate, |
| 220 int input_frames_per_buffer, | 236 int input_frames_per_buffer, |
| 221 int output_sample_rate, | 237 int output_sample_rate, |
| 222 int output_frames_per_buffer) { | 238 int output_frames_per_buffer) { |
| 223 // Audio Environment setup. | 239 // Audio Environment setup. |
| 224 source_params_.Reset(kInputFormat, | 240 source_params_.Reset(kInputFormat, |
| 225 kInputChannelLayout, | 241 kInputChannelLayout, |
| 226 input_sample_rate, | 242 input_sample_rate, |
| 227 kInputBitsPerSample, | 243 kInputBitsPerSample, |
| 228 input_frames_per_buffer); | 244 input_frames_per_buffer); |
| 229 sink_params_.Reset(kOutputFormat, | 245 sink_params_.Reset(kOutputFormat, |
| 230 kOutputChannelLayout, | 246 kOutputChannelLayout, |
| 231 output_sample_rate, | 247 output_sample_rate, |
| 232 kOutputBitsPerSample, | 248 kOutputBitsPerSample, |
| 233 output_frames_per_buffer); | 249 output_frames_per_buffer); |
| 234 source_bus_ = | 250 source_bus_ = |
| 235 media::AudioBus::Create(kInputChannels, input_frames_per_buffer); | 251 media::AudioBus::Create(kInputChannels, input_frames_per_buffer); |
| 236 source_bus_->Zero(); | 252 source_bus_->Zero(); |
| 237 first_frame_capture_time_ = base::TimeTicks::Now(); | 253 first_frame_capture_time_ = base::TimeTicks::Now(); |
| 238 sample_frames_captured_ = 0; | 254 sample_frames_captured_ = 0; |
| 239 | 255 |
| 240 // Prepare the track and audio source. | 256 // Prepare the track and audio source. |
| 241 blink::WebMediaStreamTrack blink_track; | 257 PrepareBlinkTrackOfType(MEDIA_DEVICE_AUDIO_CAPTURE, &blink_track_); |
| 242 PrepareBlinkTrackOfType(MEDIA_DEVICE_AUDIO_CAPTURE, &blink_track); | 258 blink_source_ = blink_track_.source(); |
| 243 | 259 static_cast<TestDrivenAudioSource*>( |
| 244 // Get the native track from the blink track and initialize. | 260 MediaStreamAudioSource::From(blink_source_))->SetFormat(source_params_); |
| 245 native_track_ = | |
| 246 static_cast<WebRtcLocalAudioTrack*>(blink_track.getExtraData()); | |
| 247 native_track_->OnSetFormat(source_params_); | |
| 248 | 261 |
| 249 // Create and initialize the consumer. | 262 // Create and initialize the consumer. |
| 250 recognizer_.reset(new FakeSpeechRecognizer()); | 263 recognizer_.reset(new FakeSpeechRecognizer()); |
| 251 base::SharedMemoryHandle foreign_memory_handle; | 264 base::SharedMemoryHandle foreign_memory_handle; |
| 252 recognizer_->Initialize(blink_track, sink_params_, &foreign_memory_handle); | 265 recognizer_->Initialize(blink_track_, sink_params_, &foreign_memory_handle); |
| 253 | 266 |
| 254 // Create the producer. | 267 // Create the producer. |
| 255 std::unique_ptr<base::SyncSocket> sending_socket( | 268 std::unique_ptr<base::SyncSocket> sending_socket( |
| 256 recognizer_->sending_socket()); | 269 recognizer_->sending_socket()); |
| 257 speech_audio_sink_.reset(new SpeechRecognitionAudioSink( | 270 speech_audio_sink_.reset(new SpeechRecognitionAudioSink( |
| 258 blink_track, sink_params_, foreign_memory_handle, | 271 blink_track_, sink_params_, foreign_memory_handle, |
| 259 std::move(sending_socket), | 272 std::move(sending_socket), |
| 260 base::Bind(&SpeechRecognitionAudioSinkTest::StoppedCallback, | 273 base::Bind(&SpeechRecognitionAudioSinkTest::StoppedCallback, |
| 261 base::Unretained(this)))); | 274 base::Unretained(this)))); |
| 262 | 275 |
| 263 // Return number of buffers needed to trigger resampling and consumption. | 276 // Return number of buffers needed to trigger resampling and consumption. |
| 264 return static_cast<uint32_t>(std::ceil( | 277 return static_cast<uint32_t>(std::ceil( |
| 265 static_cast<double>(output_frames_per_buffer * input_sample_rate) / | 278 static_cast<double>(output_frames_per_buffer * input_sample_rate) / |
| 266 (input_frames_per_buffer * output_sample_rate))); | 279 (input_frames_per_buffer * output_sample_rate))); |
| 267 } | 280 } |
| 268 | 281 |
| 269 // Mock callback expected to be called when the track is stopped. | 282 // Mock callback expected to be called when the track is stopped. |
| 270 MOCK_METHOD0(StoppedCallback, void()); | 283 MOCK_METHOD0(StoppedCallback, void()); |
| 271 | 284 |
| 272 protected: | 285 protected: |
| 273 // Prepares a blink track of a given MediaStreamType and attaches the native | 286 // Prepares a blink track of a given MediaStreamType and attaches the native |
| 274 // track which can be used to capture audio data and pass it to the producer. | 287 // track which can be used to capture audio data and pass it to the producer. |
| 275 static void PrepareBlinkTrackOfType( | 288 void PrepareBlinkTrackOfType(const MediaStreamType device_type, |
| 276 const MediaStreamType device_type, | 289 blink::WebMediaStreamTrack* blink_track) { |
| 277 blink::WebMediaStreamTrack* blink_track) { | 290 blink::WebMediaStreamSource blink_source; |
| 278 scoped_refptr<WebRtcLocalAudioTrackAdapter> adapter( | 291 blink_source.initialize(blink::WebString::fromUTF8("dummy_source_id"), |
| 279 WebRtcLocalAudioTrackAdapter::Create(std::string(), NULL)); | 292 blink::WebMediaStreamSource::TypeAudio, |
| 280 std::unique_ptr<WebRtcLocalAudioTrack> native_track( | 293 blink::WebString::fromUTF8("dummy_source_name"), |
| 281 new WebRtcLocalAudioTrack(adapter.get())); | 294 false /* remote */, true /* readonly */); |
| 282 blink::WebMediaStreamSource blink_audio_source; | 295 TestDrivenAudioSource* const audio_source = new TestDrivenAudioSource(); |
| 283 blink_audio_source.initialize(base::UTF8ToUTF16("dummy_source_id"), | 296 audio_source->SetDeviceInfo( |
| 284 blink::WebMediaStreamSource::TypeAudio, | 297 StreamDeviceInfo(device_type, "Mock device", "mock_device_id")); |
| 285 base::UTF8ToUTF16("dummy_source_name"), | 298 blink_source.setExtraData(audio_source); // Takes ownership. |
| 286 false /* remote */, true /* readonly */); | 299 |
| 287 MediaStreamSource::SourceStoppedCallback cb; | |
| 288 blink_audio_source.setExtraData(new MediaStreamAudioSource( | |
| 289 -1, StreamDeviceInfo(device_type, "Mock device", "mock_device_id"), cb, | |
| 290 nullptr)); | |
| 291 blink_track->initialize(blink::WebString::fromUTF8("dummy_track"), | 300 blink_track->initialize(blink::WebString::fromUTF8("dummy_track"), |
| 292 blink_audio_source); | 301 blink_source); |
| 293 blink_track->setExtraData(native_track.release()); | 302 ASSERT_TRUE(audio_source->ConnectToTrack(*blink_track)); |
| 294 } | 303 } |
| 295 | 304 |
| 296 // Emulates an audio capture device capturing data from the source. | 305 // Emulates an audio capture device capturing data from the source. |
| 297 inline void CaptureAudio(const uint32_t buffers) { | 306 inline void CaptureAudio(const uint32_t buffers) { |
| 298 for (uint32_t i = 0; i < buffers; ++i) { | 307 for (uint32_t i = 0; i < buffers; ++i) { |
| 299 const base::TimeTicks estimated_capture_time = first_frame_capture_time_ + | 308 const base::TimeTicks estimated_capture_time = first_frame_capture_time_ + |
| 300 (sample_frames_captured_ * base::TimeDelta::FromSeconds(1) / | 309 (sample_frames_captured_ * base::TimeDelta::FromSeconds(1) / |
| 301 source_params_.sample_rate()); | 310 source_params_.sample_rate()); |
| 302 native_track()->Capture(*source_bus_, estimated_capture_time); | 311 static_cast<TestDrivenAudioSource*>( |
| 312 MediaStreamAudioSource::From(blink_source_)) |
| 313 ->DeliverDataToTracks(*source_bus_, estimated_capture_time); |
| 303 sample_frames_captured_ += source_bus_->frames(); | 314 sample_frames_captured_ += source_bus_->frames(); |
| 304 } | 315 } |
| 305 } | 316 } |
| 306 | 317 |
| 307 // Used to simulate a problem with sockets. | 318 // Used to simulate a problem with sockets. |
| 308 void SetFailureModeOnForeignSocket(bool in_failure_mode) { | 319 void SetFailureModeOnForeignSocket(bool in_failure_mode) { |
| 309 recognizer()->sending_socket()->SetFailureMode(in_failure_mode); | 320 recognizer()->sending_socket()->SetFailureMode(in_failure_mode); |
| 310 } | 321 } |
| 311 | 322 |
| 312 // Helper method for verifying captured audio data has been consumed. | 323 // Helper method for verifying captured audio data has been consumed. |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 346 << ")"; | 357 << ")"; |
| 347 } | 358 } |
| 348 } | 359 } |
| 349 | 360 |
| 350 media::AudioBus* source_bus() const { return source_bus_.get(); } | 361 media::AudioBus* source_bus() const { return source_bus_.get(); } |
| 351 | 362 |
| 352 FakeSpeechRecognizer* recognizer() const { return recognizer_.get(); } | 363 FakeSpeechRecognizer* recognizer() const { return recognizer_.get(); } |
| 353 | 364 |
| 354 const media::AudioParameters& sink_params() const { return sink_params_; } | 365 const media::AudioParameters& sink_params() const { return sink_params_; } |
| 355 | 366 |
| 356 WebRtcLocalAudioTrack* native_track() const { return native_track_; } | 367 MediaStreamAudioTrack* native_track() const { |
| 368 return MediaStreamAudioTrack::From(blink_track_); |
| 369 } |
| 357 | 370 |
| 358 private: | 371 private: |
| 372 MockPeerConnectionDependencyFactory mock_dependency_factory_; |
| 373 |
| 359 // Producer. | 374 // Producer. |
| 360 std::unique_ptr<SpeechRecognitionAudioSink> speech_audio_sink_; | 375 std::unique_ptr<SpeechRecognitionAudioSink> speech_audio_sink_; |
| 361 | 376 |
| 362 // Consumer. | 377 // Consumer. |
| 363 std::unique_ptr<FakeSpeechRecognizer> recognizer_; | 378 std::unique_ptr<FakeSpeechRecognizer> recognizer_; |
| 364 | 379 |
| 365 // Audio related members. | 380 // Audio related members. |
| 366 std::unique_ptr<media::AudioBus> source_bus_; | 381 std::unique_ptr<media::AudioBus> source_bus_; |
| 367 media::AudioParameters source_params_; | 382 media::AudioParameters source_params_; |
| 368 media::AudioParameters sink_params_; | 383 media::AudioParameters sink_params_; |
| 369 WebRtcLocalAudioTrack* native_track_; | 384 blink::WebMediaStreamSource blink_source_; |
| 385 blink::WebMediaStreamTrack blink_track_; |
| 370 | 386 |
| 371 base::TimeTicks first_frame_capture_time_; | 387 base::TimeTicks first_frame_capture_time_; |
| 372 int64_t sample_frames_captured_; | 388 int64_t sample_frames_captured_; |
| 373 | 389 |
| 374 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSinkTest); | 390 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSinkTest); |
| 375 }; | 391 }; |
| 376 | 392 |
| 377 // Not all types of tracks are supported. This test checks if that policy is | 393 // Not all types of tracks are supported. This test checks if that policy is |
| 378 // implemented correctly. | 394 // implemented correctly. |
| 379 TEST_F(SpeechRecognitionAudioSinkTest, CheckIsSupportedAudioTrack) { | 395 TEST_F(SpeechRecognitionAudioSinkTest, CheckIsSupportedAudioTrack) { |
| (...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 521 const uint32_t buffers_per_notification = Initialize(44100, 441, 16000, 1600); | 537 const uint32_t buffers_per_notification = Initialize(44100, 441, 16000, 1600); |
| 522 AssertConsumedBuffers(0U); | 538 AssertConsumedBuffers(0U); |
| 523 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); | 539 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); |
| 524 EXPECT_CALL(*this, StoppedCallback()).Times(1); | 540 EXPECT_CALL(*this, StoppedCallback()).Times(1); |
| 525 | 541 |
| 526 native_track()->Stop(); | 542 native_track()->Stop(); |
| 527 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); | 543 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); |
| 528 } | 544 } |
| 529 | 545 |
| 530 } // namespace content | 546 } // namespace content |
| OLD | NEW |