OLD | NEW |
1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media/media_stream_audio_processor.h" | 5 #include "content/renderer/media/media_stream_audio_processor.h" |
6 | 6 |
7 #include "base/command_line.h" | 7 #include "base/command_line.h" |
8 #include "base/debug/trace_event.h" | 8 #include "base/debug/trace_event.h" |
9 #include "base/metrics/field_trial.h" | 9 #include "base/metrics/field_trial.h" |
10 #include "base/metrics/histogram.h" | 10 #include "base/metrics/histogram.h" |
11 #include "content/public/common/content_switches.h" | 11 #include "content/public/common/content_switches.h" |
12 #include "content/renderer/media/media_stream_audio_processor_options.h" | 12 #include "content/renderer/media/media_stream_audio_processor_options.h" |
13 #include "content/renderer/media/rtc_media_constraints.h" | 13 #include "content/renderer/media/rtc_media_constraints.h" |
14 #include "content/renderer/media/webrtc_audio_device_impl.h" | 14 #include "content/renderer/media/webrtc_audio_device_impl.h" |
15 #include "media/audio/audio_parameters.h" | 15 #include "media/audio/audio_parameters.h" |
16 #include "media/base/audio_converter.h" | 16 #include "media/base/audio_converter.h" |
17 #include "media/base/audio_fifo.h" | 17 #include "media/base/audio_fifo.h" |
18 #include "media/base/channel_layout.h" | 18 #include "media/base/channel_layout.h" |
19 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" | 19 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" |
20 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface
.h" | 20 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface
.h" |
21 #include "third_party/webrtc/modules/audio_processing/typing_detection.h" | 21 #include "third_party/webrtc/modules/audio_processing/typing_detection.h" |
22 | 22 |
23 namespace content { | 23 namespace content { |
24 | 24 |
25 namespace { | 25 namespace { |
26 | 26 |
27 using webrtc::AudioProcessing; | 27 using webrtc::AudioProcessing; |
28 using webrtc::MediaConstraintsInterface; | |
29 | 28 |
30 #if defined(OS_ANDROID) | 29 #if defined(OS_ANDROID) |
31 const int kAudioProcessingSampleRate = 16000; | 30 const int kAudioProcessingSampleRate = 16000; |
32 #else | 31 #else |
33 const int kAudioProcessingSampleRate = 32000; | 32 const int kAudioProcessingSampleRate = 32000; |
34 #endif | 33 #endif |
35 const int kAudioProcessingNumberOfChannels = 1; | 34 const int kAudioProcessingNumberOfChannels = 1; |
36 | 35 |
37 const int kMaxNumberOfBuffersInFifo = 2; | 36 const int kMaxNumberOfBuffersInFifo = 2; |
38 | 37 |
(...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
158 bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { | 157 bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { |
159 const std::string group_name = | 158 const std::string group_name = |
160 base::FieldTrialList::FindFullName("MediaStreamAudioTrackProcessing"); | 159 base::FieldTrialList::FindFullName("MediaStreamAudioTrackProcessing"); |
161 return group_name == "Enabled" || CommandLine::ForCurrentProcess()->HasSwitch( | 160 return group_name == "Enabled" || CommandLine::ForCurrentProcess()->HasSwitch( |
162 switches::kEnableAudioTrackProcessing); | 161 switches::kEnableAudioTrackProcessing); |
163 } | 162 } |
164 | 163 |
165 MediaStreamAudioProcessor::MediaStreamAudioProcessor( | 164 MediaStreamAudioProcessor::MediaStreamAudioProcessor( |
166 const blink::WebMediaConstraints& constraints, | 165 const blink::WebMediaConstraints& constraints, |
167 int effects, | 166 int effects, |
168 MediaStreamType type, | |
169 WebRtcPlayoutDataSource* playout_data_source) | 167 WebRtcPlayoutDataSource* playout_data_source) |
170 : render_delay_ms_(0), | 168 : render_delay_ms_(0), |
171 playout_data_source_(playout_data_source), | 169 playout_data_source_(playout_data_source), |
172 audio_mirroring_(false), | 170 audio_mirroring_(false), |
173 typing_detected_(false) { | 171 typing_detected_(false) { |
174 capture_thread_checker_.DetachFromThread(); | 172 capture_thread_checker_.DetachFromThread(); |
175 render_thread_checker_.DetachFromThread(); | 173 render_thread_checker_.DetachFromThread(); |
176 InitializeAudioProcessingModule(constraints, effects, type); | 174 InitializeAudioProcessingModule(constraints, effects); |
177 } | 175 } |
178 | 176 |
179 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { | 177 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { |
180 DCHECK(main_thread_checker_.CalledOnValidThread()); | 178 DCHECK(main_thread_checker_.CalledOnValidThread()); |
181 StopAudioProcessing(); | 179 StopAudioProcessing(); |
182 } | 180 } |
183 | 181 |
184 void MediaStreamAudioProcessor::OnCaptureFormatChanged( | 182 void MediaStreamAudioProcessor::OnCaptureFormatChanged( |
185 const media::AudioParameters& source_params) { | 183 const media::AudioParameters& source_params) { |
186 DCHECK(main_thread_checker_.CalledOnValidThread()); | 184 DCHECK(main_thread_checker_.CalledOnValidThread()); |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
277 render_converter_.reset(); | 275 render_converter_.reset(); |
278 } | 276 } |
279 | 277 |
280 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { | 278 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { |
281 stats->typing_noise_detected = | 279 stats->typing_noise_detected = |
282 (base::subtle::Acquire_Load(&typing_detected_) != false); | 280 (base::subtle::Acquire_Load(&typing_detected_) != false); |
283 GetAecStats(audio_processing_.get(), stats); | 281 GetAecStats(audio_processing_.get(), stats); |
284 } | 282 } |
285 | 283 |
286 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( | 284 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( |
287 const blink::WebMediaConstraints& constraints, int effects, | 285 const blink::WebMediaConstraints& constraints, int effects) { |
288 MediaStreamType type) { | |
289 DCHECK(!audio_processing_); | 286 DCHECK(!audio_processing_); |
290 | 287 |
291 RTCMediaConstraints native_constraints(constraints); | 288 MediaAudioConstraints audio_constraints(constraints, effects); |
292 | 289 |
293 // Audio mirroring can be enabled even though audio processing is otherwise | 290 // Audio mirroring can be enabled even though audio processing is otherwise |
294 // disabled. | 291 // disabled. |
295 audio_mirroring_ = GetPropertyFromConstraints( | 292 audio_mirroring_ = audio_constraints.GetProperty( |
296 &native_constraints, webrtc::MediaConstraintsInterface::kAudioMirroring); | 293 MediaAudioConstraints::kGoogAudioMirroring); |
297 | 294 |
298 if (!IsAudioTrackProcessingEnabled()) { | 295 if (!IsAudioTrackProcessingEnabled()) { |
299 RecordProcessingState(AUDIO_PROCESSING_IN_WEBRTC); | 296 RecordProcessingState(AUDIO_PROCESSING_IN_WEBRTC); |
300 return; | 297 return; |
301 } | 298 } |
302 | 299 |
303 // Only apply the fixed constraints for gUM of MEDIA_DEVICE_AUDIO_CAPTURE. | 300 // |kEchoCancellation| is used as a master control on enabling/disabling |
304 DCHECK(IsAudioMediaType(type)); | 301 // the audio processing. |
305 if (type == MEDIA_DEVICE_AUDIO_CAPTURE) | 302 // If |kEchoCancellation| is specified in |audio_constraints|, it will use |
306 ApplyFixedAudioConstraints(&native_constraints); | 303 // the defined value there, otherwise use the default value defined by |
307 | 304 // kDefaultAudioConstraints in media_stream_audio_processor_options.cc. |
308 if (effects & media::AudioParameters::ECHO_CANCELLER) { | 305 const bool echo_cancellation = audio_constraints.GetProperty( |
309 // If platform echo canceller is enabled, disable the software AEC. | 306 MediaAudioConstraints::kEchoCancellation); |
310 native_constraints.AddMandatory( | 307 if (!echo_cancellation) { |
311 MediaConstraintsInterface::kEchoCancellation, | 308 RecordProcessingState(AUDIO_PROCESSING_DISABLED); |
312 MediaConstraintsInterface::kValueFalse, true); | 309 return; |
313 } | 310 } |
314 | 311 |
315 #if defined(OS_IOS) | 312 #if defined(OS_IOS) |
316 // On iOS, VPIO provides built-in AEC and AGC. | 313 // On iOS, VPIO provides built-in AEC and AGC. |
317 const bool enable_aec = false; | 314 const bool goog_aec = false; |
318 const bool enable_agc = false; | 315 const bool goog_agc = false; |
319 #else | 316 #else |
320 const bool enable_aec = GetPropertyFromConstraints( | 317 // TODO(xians): goog_aec should just be echo_cancellation. |
321 &native_constraints, MediaConstraintsInterface::kEchoCancellation); | 318 const bool goog_aec = audio_constraints.GetProperty( |
322 const bool enable_agc = GetPropertyFromConstraints( | 319 MediaAudioConstraints::kGoogEchoCancellation); |
323 &native_constraints, webrtc::MediaConstraintsInterface::kAutoGainControl); | 320 const bool goog_agc = audio_constraints.GetProperty( |
| 321 MediaAudioConstraints::kGoogAutoGainControl); |
324 #endif | 322 #endif |
325 | 323 |
326 #if defined(OS_IOS) || defined(OS_ANDROID) | 324 #if defined(OS_IOS) || defined(OS_ANDROID) |
327 const bool enable_experimental_aec = false; | 325 const bool goog_experimental_aec = false; |
328 const bool enable_typing_detection = false; | 326 const bool goog_typing_detection = false; |
329 #else | 327 #else |
330 const bool enable_experimental_aec = GetPropertyFromConstraints( | 328 const bool goog_experimental_aec = audio_constraints.GetProperty( |
331 &native_constraints, | 329 MediaAudioConstraints::kGoogExperimentalEchoCancellation); |
332 MediaConstraintsInterface::kExperimentalEchoCancellation); | 330 const bool goog_typing_detection = audio_constraints.GetProperty( |
333 const bool enable_typing_detection = GetPropertyFromConstraints( | 331 MediaAudioConstraints::kGoogTypingNoiseDetection); |
334 &native_constraints, MediaConstraintsInterface::kTypingNoiseDetection); | |
335 #endif | 332 #endif |
336 | 333 |
337 const bool enable_ns = GetPropertyFromConstraints( | 334 const bool goog_ns = audio_constraints.GetProperty( |
338 &native_constraints, MediaConstraintsInterface::kNoiseSuppression); | 335 MediaAudioConstraints::kGoogNoiseSuppression); |
339 const bool enable_experimental_ns = GetPropertyFromConstraints( | 336 const bool goog_experimental_ns = audio_constraints.GetProperty( |
340 &native_constraints, | 337 MediaAudioConstraints::kGoogExperimentalNoiseSuppression); |
341 MediaConstraintsInterface::kExperimentalNoiseSuppression); | 338 const bool goog_high_pass_filter = audio_constraints.GetProperty( |
342 const bool enable_high_pass_filter = GetPropertyFromConstraints( | 339 MediaAudioConstraints::kGoogHighpassFilter); |
343 &native_constraints, MediaConstraintsInterface::kHighpassFilter); | |
344 | 340 |
345 // Return immediately if no audio processing component is enabled. | 341 // Return immediately if no goog constraint is enabled. |
346 if (!enable_aec && !enable_experimental_aec && !enable_ns && | 342 if (!goog_aec && !goog_experimental_aec && !goog_ns && |
347 !enable_high_pass_filter && !enable_typing_detection && !enable_agc && | 343 !goog_high_pass_filter && !goog_typing_detection && |
348 !enable_experimental_ns) { | 344 !goog_agc && !goog_experimental_ns) { |
349 RecordProcessingState(AUDIO_PROCESSING_DISABLED); | 345 RecordProcessingState(AUDIO_PROCESSING_DISABLED); |
350 return; | 346 return; |
351 } | 347 } |
352 | 348 |
353 // Create and configure the webrtc::AudioProcessing. | 349 // Create and configure the webrtc::AudioProcessing. |
354 audio_processing_.reset(webrtc::AudioProcessing::Create(0)); | 350 audio_processing_.reset(webrtc::AudioProcessing::Create(0)); |
355 // TODO(ajm): Replace with AudioProcessing::Initialize() when this rolls to | 351 // TODO(ajm): Replace with AudioProcessing::Initialize() when this rolls to |
356 // Chromium: http://review.webrtc.org/9919004/ | 352 // Chromium: http://review.webrtc.org/9919004/ |
357 CHECK_EQ(0, | 353 CHECK_EQ(0, |
358 audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate)); | 354 audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate)); |
359 | 355 |
360 // Enable the audio processing components. | 356 // Enable the audio processing components. |
361 if (enable_aec) { | 357 if (goog_aec) { |
362 EnableEchoCancellation(audio_processing_.get()); | 358 EnableEchoCancellation(audio_processing_.get()); |
363 if (enable_experimental_aec) | 359 |
| 360 if (goog_experimental_aec) |
364 EnableExperimentalEchoCancellation(audio_processing_.get()); | 361 EnableExperimentalEchoCancellation(audio_processing_.get()); |
365 | 362 |
366 if (playout_data_source_) | 363 if (playout_data_source_) |
367 playout_data_source_->AddPlayoutSink(this); | 364 playout_data_source_->AddPlayoutSink(this); |
368 } | 365 } |
369 | 366 |
370 if (enable_ns) | 367 if (goog_ns) |
371 EnableNoiseSuppression(audio_processing_.get()); | 368 EnableNoiseSuppression(audio_processing_.get()); |
372 | 369 |
373 if (enable_experimental_ns) | 370 if (goog_experimental_ns) |
374 EnableExperimentalNoiseSuppression(audio_processing_.get()); | 371 EnableExperimentalNoiseSuppression(audio_processing_.get()); |
375 | 372 |
376 if (enable_high_pass_filter) | 373 if (goog_high_pass_filter) |
377 EnableHighPassFilter(audio_processing_.get()); | 374 EnableHighPassFilter(audio_processing_.get()); |
378 | 375 |
379 if (enable_typing_detection) { | 376 if (goog_typing_detection) { |
380 // TODO(xians): Remove this |typing_detector_| after the typing suppression | 377 // TODO(xians): Remove this |typing_detector_| after the typing suppression |
381 // is enabled by default. | 378 // is enabled by default. |
382 typing_detector_.reset(new webrtc::TypingDetection()); | 379 typing_detector_.reset(new webrtc::TypingDetection()); |
383 EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); | 380 EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); |
384 } | 381 } |
385 | 382 |
386 if (enable_agc) | 383 if (goog_agc) |
387 EnableAutomaticGainControl(audio_processing_.get()); | 384 EnableAutomaticGainControl(audio_processing_.get()); |
388 | 385 |
389 RecordProcessingState(AUDIO_PROCESSING_ENABLED); | 386 RecordProcessingState(AUDIO_PROCESSING_ENABLED); |
390 } | 387 } |
391 | 388 |
392 void MediaStreamAudioProcessor::InitializeCaptureConverter( | 389 void MediaStreamAudioProcessor::InitializeCaptureConverter( |
393 const media::AudioParameters& source_params) { | 390 const media::AudioParameters& source_params) { |
394 DCHECK(main_thread_checker_.CalledOnValidThread()); | 391 DCHECK(main_thread_checker_.CalledOnValidThread()); |
395 DCHECK(source_params.IsValid()); | 392 DCHECK(source_params.IsValid()); |
396 | 393 |
(...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
509 | 506 |
510 StopAecDump(); | 507 StopAecDump(); |
511 | 508 |
512 if (playout_data_source_) | 509 if (playout_data_source_) |
513 playout_data_source_->RemovePlayoutSink(this); | 510 playout_data_source_->RemovePlayoutSink(this); |
514 | 511 |
515 audio_processing_.reset(); | 512 audio_processing_.reset(); |
516 } | 513 } |
517 | 514 |
518 } // namespace content | 515 } // namespace content |
OLD | NEW |