OLD | NEW |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media/speech_recognition_audio_sink.h" | 5 #include "content/renderer/media/speech_recognition_audio_sink.h" |
6 | 6 |
7 #include <stddef.h> | 7 #include <stddef.h> |
8 #include <stdint.h> | 8 #include <stdint.h> |
9 #include <string.h> | 9 #include <string.h> |
10 #include <utility> | 10 #include <utility> |
11 | 11 |
12 #include "base/bind.h" | 12 #include "base/bind.h" |
13 #include "base/macros.h" | 13 #include "base/macros.h" |
14 #include "base/strings/utf_string_conversions.h" | |
15 #include "content/renderer/media/media_stream_audio_source.h" | 14 #include "content/renderer/media/media_stream_audio_source.h" |
16 #include "content/renderer/media/webrtc/webrtc_local_audio_track_adapter.h" | 15 #include "content/renderer/media/media_stream_audio_track.h" |
17 #include "content/renderer/media/webrtc_local_audio_track.h" | 16 #include "content/renderer/media/webrtc/mock_peer_connection_dependency_factory.
h" |
18 #include "media/base/audio_bus.h" | 17 #include "media/base/audio_bus.h" |
19 #include "media/base/audio_parameters.h" | 18 #include "media/base/audio_parameters.h" |
20 #include "testing/gmock/include/gmock/gmock.h" | 19 #include "testing/gmock/include/gmock/gmock.h" |
21 #include "testing/gtest/include/gtest/gtest.h" | 20 #include "testing/gtest/include/gtest/gtest.h" |
22 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" | 21 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" |
| 22 #include "third_party/WebKit/public/platform/WebString.h" |
23 #include "third_party/WebKit/public/web/WebHeap.h" | 23 #include "third_party/WebKit/public/web/WebHeap.h" |
24 | 24 |
25 namespace { | 25 namespace { |
26 | 26 |
27 // Supported speech recognition audio parameters. | 27 // Supported speech recognition audio parameters. |
28 const int kSpeechRecognitionSampleRate = 16000; | 28 const int kSpeechRecognitionSampleRate = 16000; |
29 const int kSpeechRecognitionFramesPerBuffer = 1600; | 29 const int kSpeechRecognitionFramesPerBuffer = 1600; |
30 | 30 |
31 // Input audio format. | 31 // Input audio format. |
32 const media::AudioParameters::Format kInputFormat = | 32 const media::AudioParameters::Format kInputFormat = |
(...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
197 // Audio bus wrapping the shared memory from the renderer. | 197 // Audio bus wrapping the shared memory from the renderer. |
198 std::unique_ptr<media::AudioBus> audio_track_bus_; | 198 std::unique_ptr<media::AudioBus> audio_track_bus_; |
199 | 199 |
200 DISALLOW_COPY_AND_ASSIGN(FakeSpeechRecognizer); | 200 DISALLOW_COPY_AND_ASSIGN(FakeSpeechRecognizer); |
201 }; | 201 }; |
202 | 202 |
203 } // namespace | 203 } // namespace |
204 | 204 |
205 namespace content { | 205 namespace content { |
206 | 206 |
| 207 namespace { |
| 208 |
| 209 class TestDrivenAudioSource : public MediaStreamAudioSource { |
| 210 public: |
| 211 TestDrivenAudioSource() : MediaStreamAudioSource(true) {} |
| 212 ~TestDrivenAudioSource() final {} |
| 213 |
| 214 // Expose protected methods as public for testing. |
| 215 using MediaStreamAudioSource::SetFormat; |
| 216 using MediaStreamAudioSource::DeliverDataToTracks; |
| 217 }; |
| 218 |
| 219 } // namespace |
| 220 |
207 class SpeechRecognitionAudioSinkTest : public testing::Test { | 221 class SpeechRecognitionAudioSinkTest : public testing::Test { |
208 public: | 222 public: |
209 SpeechRecognitionAudioSinkTest() {} | 223 SpeechRecognitionAudioSinkTest() {} |
210 | 224 |
211 ~SpeechRecognitionAudioSinkTest() { | 225 ~SpeechRecognitionAudioSinkTest() { |
| 226 blink_source_.reset(); |
| 227 blink_track_.reset(); |
212 speech_audio_sink_.reset(); | 228 speech_audio_sink_.reset(); |
213 blink::WebHeap::collectAllGarbageForTesting(); | 229 blink::WebHeap::collectAllGarbageForTesting(); |
214 } | 230 } |
215 | 231 |
216 // Initializes the producer and consumer with specified audio parameters. | 232 // Initializes the producer and consumer with specified audio parameters. |
217 // Returns the minimal number of input audio buffers which need to be captured | 233 // Returns the minimal number of input audio buffers which need to be captured |
218 // before they get sent to the consumer. | 234 // before they get sent to the consumer. |
219 uint32_t Initialize(int input_sample_rate, | 235 uint32_t Initialize(int input_sample_rate, |
220 int input_frames_per_buffer, | 236 int input_frames_per_buffer, |
221 int output_sample_rate, | 237 int output_sample_rate, |
222 int output_frames_per_buffer) { | 238 int output_frames_per_buffer) { |
223 // Audio Environment setup. | 239 // Audio Environment setup. |
224 source_params_.Reset(kInputFormat, | 240 source_params_.Reset(kInputFormat, |
225 kInputChannelLayout, | 241 kInputChannelLayout, |
226 input_sample_rate, | 242 input_sample_rate, |
227 kInputBitsPerSample, | 243 kInputBitsPerSample, |
228 input_frames_per_buffer); | 244 input_frames_per_buffer); |
229 sink_params_.Reset(kOutputFormat, | 245 sink_params_.Reset(kOutputFormat, |
230 kOutputChannelLayout, | 246 kOutputChannelLayout, |
231 output_sample_rate, | 247 output_sample_rate, |
232 kOutputBitsPerSample, | 248 kOutputBitsPerSample, |
233 output_frames_per_buffer); | 249 output_frames_per_buffer); |
234 source_bus_ = | 250 source_bus_ = |
235 media::AudioBus::Create(kInputChannels, input_frames_per_buffer); | 251 media::AudioBus::Create(kInputChannels, input_frames_per_buffer); |
236 source_bus_->Zero(); | 252 source_bus_->Zero(); |
237 first_frame_capture_time_ = base::TimeTicks::Now(); | 253 first_frame_capture_time_ = base::TimeTicks::Now(); |
238 sample_frames_captured_ = 0; | 254 sample_frames_captured_ = 0; |
239 | 255 |
240 // Prepare the track and audio source. | 256 // Prepare the track and audio source. |
241 blink::WebMediaStreamTrack blink_track; | 257 PrepareBlinkTrackOfType(MEDIA_DEVICE_AUDIO_CAPTURE, &blink_track_); |
242 PrepareBlinkTrackOfType(MEDIA_DEVICE_AUDIO_CAPTURE, &blink_track); | 258 blink_source_ = blink_track_.source(); |
243 | 259 static_cast<TestDrivenAudioSource*>( |
244 // Get the native track from the blink track and initialize. | 260 MediaStreamAudioSource::From(blink_source_))->SetFormat(source_params_); |
245 native_track_ = | |
246 static_cast<WebRtcLocalAudioTrack*>(blink_track.getExtraData()); | |
247 native_track_->OnSetFormat(source_params_); | |
248 | 261 |
249 // Create and initialize the consumer. | 262 // Create and initialize the consumer. |
250 recognizer_.reset(new FakeSpeechRecognizer()); | 263 recognizer_.reset(new FakeSpeechRecognizer()); |
251 base::SharedMemoryHandle foreign_memory_handle; | 264 base::SharedMemoryHandle foreign_memory_handle; |
252 recognizer_->Initialize(blink_track, sink_params_, &foreign_memory_handle); | 265 recognizer_->Initialize(blink_track_, sink_params_, &foreign_memory_handle); |
253 | 266 |
254 // Create the producer. | 267 // Create the producer. |
255 std::unique_ptr<base::SyncSocket> sending_socket( | 268 std::unique_ptr<base::SyncSocket> sending_socket( |
256 recognizer_->sending_socket()); | 269 recognizer_->sending_socket()); |
257 speech_audio_sink_.reset(new SpeechRecognitionAudioSink( | 270 speech_audio_sink_.reset(new SpeechRecognitionAudioSink( |
258 blink_track, sink_params_, foreign_memory_handle, | 271 blink_track_, sink_params_, foreign_memory_handle, |
259 std::move(sending_socket), | 272 std::move(sending_socket), |
260 base::Bind(&SpeechRecognitionAudioSinkTest::StoppedCallback, | 273 base::Bind(&SpeechRecognitionAudioSinkTest::StoppedCallback, |
261 base::Unretained(this)))); | 274 base::Unretained(this)))); |
262 | 275 |
263 // Return number of buffers needed to trigger resampling and consumption. | 276 // Return number of buffers needed to trigger resampling and consumption. |
264 return static_cast<uint32_t>(std::ceil( | 277 return static_cast<uint32_t>(std::ceil( |
265 static_cast<double>(output_frames_per_buffer * input_sample_rate) / | 278 static_cast<double>(output_frames_per_buffer * input_sample_rate) / |
266 (input_frames_per_buffer * output_sample_rate))); | 279 (input_frames_per_buffer * output_sample_rate))); |
267 } | 280 } |
268 | 281 |
269 // Mock callback expected to be called when the track is stopped. | 282 // Mock callback expected to be called when the track is stopped. |
270 MOCK_METHOD0(StoppedCallback, void()); | 283 MOCK_METHOD0(StoppedCallback, void()); |
271 | 284 |
272 protected: | 285 protected: |
273 // Prepares a blink track of a given MediaStreamType and attaches the native | 286 // Prepares a blink track of a given MediaStreamType and attaches the native |
274 // track which can be used to capture audio data and pass it to the producer. | 287 // track which can be used to capture audio data and pass it to the producer. |
275 static void PrepareBlinkTrackOfType( | 288 void PrepareBlinkTrackOfType(const MediaStreamType device_type, |
276 const MediaStreamType device_type, | 289 blink::WebMediaStreamTrack* blink_track) { |
277 blink::WebMediaStreamTrack* blink_track) { | 290 blink::WebMediaStreamSource blink_source; |
278 scoped_refptr<WebRtcLocalAudioTrackAdapter> adapter( | 291 blink_source.initialize(blink::WebString::fromUTF8("dummy_source_id"), |
279 WebRtcLocalAudioTrackAdapter::Create(std::string(), NULL)); | 292 blink::WebMediaStreamSource::TypeAudio, |
280 std::unique_ptr<WebRtcLocalAudioTrack> native_track( | 293 blink::WebString::fromUTF8("dummy_source_name"), |
281 new WebRtcLocalAudioTrack(adapter.get())); | 294 false /* remote */, true /* readonly */); |
282 blink::WebMediaStreamSource blink_audio_source; | 295 TestDrivenAudioSource* const audio_source = new TestDrivenAudioSource(); |
283 blink_audio_source.initialize(base::UTF8ToUTF16("dummy_source_id"), | 296 audio_source->SetDeviceInfo( |
284 blink::WebMediaStreamSource::TypeAudio, | 297 StreamDeviceInfo(device_type, "Mock device", "mock_device_id")); |
285 base::UTF8ToUTF16("dummy_source_name"), | 298 blink_source.setExtraData(audio_source); // Takes ownership. |
286 false /* remote */, true /* readonly */); | 299 |
287 MediaStreamSource::SourceStoppedCallback cb; | |
288 blink_audio_source.setExtraData(new MediaStreamAudioSource( | |
289 -1, StreamDeviceInfo(device_type, "Mock device", "mock_device_id"), cb, | |
290 nullptr)); | |
291 blink_track->initialize(blink::WebString::fromUTF8("dummy_track"), | 300 blink_track->initialize(blink::WebString::fromUTF8("dummy_track"), |
292 blink_audio_source); | 301 blink_source); |
293 blink_track->setExtraData(native_track.release()); | 302 ASSERT_TRUE(audio_source->ConnectToTrack(*blink_track)); |
294 } | 303 } |
295 | 304 |
296 // Emulates an audio capture device capturing data from the source. | 305 // Emulates an audio capture device capturing data from the source. |
297 inline void CaptureAudio(const uint32_t buffers) { | 306 inline void CaptureAudio(const uint32_t buffers) { |
298 for (uint32_t i = 0; i < buffers; ++i) { | 307 for (uint32_t i = 0; i < buffers; ++i) { |
299 const base::TimeTicks estimated_capture_time = first_frame_capture_time_ + | 308 const base::TimeTicks estimated_capture_time = first_frame_capture_time_ + |
300 (sample_frames_captured_ * base::TimeDelta::FromSeconds(1) / | 309 (sample_frames_captured_ * base::TimeDelta::FromSeconds(1) / |
301 source_params_.sample_rate()); | 310 source_params_.sample_rate()); |
302 native_track()->Capture(*source_bus_, estimated_capture_time); | 311 static_cast<TestDrivenAudioSource*>( |
| 312 MediaStreamAudioSource::From(blink_source_)) |
| 313 ->DeliverDataToTracks(*source_bus_, estimated_capture_time); |
303 sample_frames_captured_ += source_bus_->frames(); | 314 sample_frames_captured_ += source_bus_->frames(); |
304 } | 315 } |
305 } | 316 } |
306 | 317 |
307 // Used to simulate a problem with sockets. | 318 // Used to simulate a problem with sockets. |
308 void SetFailureModeOnForeignSocket(bool in_failure_mode) { | 319 void SetFailureModeOnForeignSocket(bool in_failure_mode) { |
309 recognizer()->sending_socket()->SetFailureMode(in_failure_mode); | 320 recognizer()->sending_socket()->SetFailureMode(in_failure_mode); |
310 } | 321 } |
311 | 322 |
312 // Helper method for verifying captured audio data has been consumed. | 323 // Helper method for verifying captured audio data has been consumed. |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
346 << ")"; | 357 << ")"; |
347 } | 358 } |
348 } | 359 } |
349 | 360 |
350 media::AudioBus* source_bus() const { return source_bus_.get(); } | 361 media::AudioBus* source_bus() const { return source_bus_.get(); } |
351 | 362 |
352 FakeSpeechRecognizer* recognizer() const { return recognizer_.get(); } | 363 FakeSpeechRecognizer* recognizer() const { return recognizer_.get(); } |
353 | 364 |
354 const media::AudioParameters& sink_params() const { return sink_params_; } | 365 const media::AudioParameters& sink_params() const { return sink_params_; } |
355 | 366 |
356 WebRtcLocalAudioTrack* native_track() const { return native_track_; } | 367 MediaStreamAudioTrack* native_track() const { |
| 368 return MediaStreamAudioTrack::From(blink_track_); |
| 369 } |
357 | 370 |
358 private: | 371 private: |
| 372 MockPeerConnectionDependencyFactory mock_dependency_factory_; |
| 373 |
359 // Producer. | 374 // Producer. |
360 std::unique_ptr<SpeechRecognitionAudioSink> speech_audio_sink_; | 375 std::unique_ptr<SpeechRecognitionAudioSink> speech_audio_sink_; |
361 | 376 |
362 // Consumer. | 377 // Consumer. |
363 std::unique_ptr<FakeSpeechRecognizer> recognizer_; | 378 std::unique_ptr<FakeSpeechRecognizer> recognizer_; |
364 | 379 |
365 // Audio related members. | 380 // Audio related members. |
366 std::unique_ptr<media::AudioBus> source_bus_; | 381 std::unique_ptr<media::AudioBus> source_bus_; |
367 media::AudioParameters source_params_; | 382 media::AudioParameters source_params_; |
368 media::AudioParameters sink_params_; | 383 media::AudioParameters sink_params_; |
369 WebRtcLocalAudioTrack* native_track_; | 384 blink::WebMediaStreamSource blink_source_; |
| 385 blink::WebMediaStreamTrack blink_track_; |
370 | 386 |
371 base::TimeTicks first_frame_capture_time_; | 387 base::TimeTicks first_frame_capture_time_; |
372 int64_t sample_frames_captured_; | 388 int64_t sample_frames_captured_; |
373 | 389 |
374 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSinkTest); | 390 DISALLOW_COPY_AND_ASSIGN(SpeechRecognitionAudioSinkTest); |
375 }; | 391 }; |
376 | 392 |
377 // Not all types of tracks are supported. This test checks if that policy is | 393 // Not all types of tracks are supported. This test checks if that policy is |
378 // implemented correctly. | 394 // implemented correctly. |
379 TEST_F(SpeechRecognitionAudioSinkTest, CheckIsSupportedAudioTrack) { | 395 TEST_F(SpeechRecognitionAudioSinkTest, CheckIsSupportedAudioTrack) { |
(...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
521 const uint32_t buffers_per_notification = Initialize(44100, 441, 16000, 1600); | 537 const uint32_t buffers_per_notification = Initialize(44100, 441, 16000, 1600); |
522 AssertConsumedBuffers(0U); | 538 AssertConsumedBuffers(0U); |
523 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); | 539 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); |
524 EXPECT_CALL(*this, StoppedCallback()).Times(1); | 540 EXPECT_CALL(*this, StoppedCallback()).Times(1); |
525 | 541 |
526 native_track()->Stop(); | 542 native_track()->Stop(); |
527 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); | 543 CaptureAudioAndAssertConsumedBuffers(buffers_per_notification, 1U); |
528 } | 544 } |
529 | 545 |
530 } // namespace content | 546 } // namespace content |
OLD | NEW |