| OLD | NEW |
| 1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "content/renderer/media/media_stream_audio_processor.h" | 5 #include "content/renderer/media/media_stream_audio_processor.h" |
| 6 | 6 |
| 7 #include "base/command_line.h" | 7 #include "base/command_line.h" |
| 8 #include "base/debug/trace_event.h" | 8 #include "base/debug/trace_event.h" |
| 9 #include "base/metrics/field_trial.h" | 9 #include "base/metrics/field_trial.h" |
| 10 #include "base/metrics/histogram.h" | 10 #include "base/metrics/histogram.h" |
| 11 #include "content/public/common/content_switches.h" | 11 #include "content/public/common/content_switches.h" |
| 12 #include "content/renderer/media/media_stream_audio_processor_options.h" | 12 #include "content/renderer/media/media_stream_audio_processor_options.h" |
| 13 #include "content/renderer/media/rtc_media_constraints.h" | 13 #include "content/renderer/media/rtc_media_constraints.h" |
| 14 #include "content/renderer/media/webrtc_audio_device_impl.h" | 14 #include "content/renderer/media/webrtc_audio_device_impl.h" |
| 15 #include "media/audio/audio_parameters.h" | 15 #include "media/audio/audio_parameters.h" |
| 16 #include "media/base/audio_converter.h" | 16 #include "media/base/audio_converter.h" |
| 17 #include "media/base/audio_fifo.h" | 17 #include "media/base/audio_fifo.h" |
| 18 #include "media/base/channel_layout.h" | 18 #include "media/base/channel_layout.h" |
| 19 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" | 19 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" |
| 20 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface
.h" | 20 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface
.h" |
| 21 #include "third_party/webrtc/modules/audio_processing/typing_detection.h" | 21 #include "third_party/webrtc/modules/audio_processing/typing_detection.h" |
| 22 | 22 |
| 23 namespace content { | 23 namespace content { |
| 24 | 24 |
| 25 namespace { | 25 namespace { |
| 26 | 26 |
| 27 using webrtc::AudioProcessing; | 27 using webrtc::AudioProcessing; |
| 28 using webrtc::MediaConstraintsInterface; | |
| 29 | 28 |
| 30 #if defined(OS_ANDROID) | 29 #if defined(OS_ANDROID) |
| 31 const int kAudioProcessingSampleRate = 16000; | 30 const int kAudioProcessingSampleRate = 16000; |
| 32 #else | 31 #else |
| 33 const int kAudioProcessingSampleRate = 32000; | 32 const int kAudioProcessingSampleRate = 32000; |
| 34 #endif | 33 #endif |
| 35 const int kAudioProcessingNumberOfChannels = 1; | 34 const int kAudioProcessingNumberOfChannels = 1; |
| 36 | 35 |
| 37 const int kMaxNumberOfBuffersInFifo = 2; | 36 const int kMaxNumberOfBuffersInFifo = 2; |
| 38 | 37 |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 158 bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { | 157 bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { |
| 159 const std::string group_name = | 158 const std::string group_name = |
| 160 base::FieldTrialList::FindFullName("MediaStreamAudioTrackProcessing"); | 159 base::FieldTrialList::FindFullName("MediaStreamAudioTrackProcessing"); |
| 161 return group_name == "Enabled" || CommandLine::ForCurrentProcess()->HasSwitch( | 160 return group_name == "Enabled" || CommandLine::ForCurrentProcess()->HasSwitch( |
| 162 switches::kEnableAudioTrackProcessing); | 161 switches::kEnableAudioTrackProcessing); |
| 163 } | 162 } |
| 164 | 163 |
| 165 MediaStreamAudioProcessor::MediaStreamAudioProcessor( | 164 MediaStreamAudioProcessor::MediaStreamAudioProcessor( |
| 166 const blink::WebMediaConstraints& constraints, | 165 const blink::WebMediaConstraints& constraints, |
| 167 int effects, | 166 int effects, |
| 168 MediaStreamType type, | |
| 169 WebRtcPlayoutDataSource* playout_data_source) | 167 WebRtcPlayoutDataSource* playout_data_source) |
| 170 : render_delay_ms_(0), | 168 : render_delay_ms_(0), |
| 171 playout_data_source_(playout_data_source), | 169 playout_data_source_(playout_data_source), |
| 172 audio_mirroring_(false), | 170 audio_mirroring_(false), |
| 173 typing_detected_(false) { | 171 typing_detected_(false) { |
| 174 capture_thread_checker_.DetachFromThread(); | 172 capture_thread_checker_.DetachFromThread(); |
| 175 render_thread_checker_.DetachFromThread(); | 173 render_thread_checker_.DetachFromThread(); |
| 176 InitializeAudioProcessingModule(constraints, effects, type); | 174 InitializeAudioProcessingModule(constraints, effects); |
| 177 } | 175 } |
| 178 | 176 |
| 179 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { | 177 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { |
| 180 DCHECK(main_thread_checker_.CalledOnValidThread()); | 178 DCHECK(main_thread_checker_.CalledOnValidThread()); |
| 181 StopAudioProcessing(); | 179 StopAudioProcessing(); |
| 182 } | 180 } |
| 183 | 181 |
| 184 void MediaStreamAudioProcessor::OnCaptureFormatChanged( | 182 void MediaStreamAudioProcessor::OnCaptureFormatChanged( |
| 185 const media::AudioParameters& source_params) { | 183 const media::AudioParameters& source_params) { |
| 186 DCHECK(main_thread_checker_.CalledOnValidThread()); | 184 DCHECK(main_thread_checker_.CalledOnValidThread()); |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 277 render_converter_.reset(); | 275 render_converter_.reset(); |
| 278 } | 276 } |
| 279 | 277 |
| 280 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { | 278 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { |
| 281 stats->typing_noise_detected = | 279 stats->typing_noise_detected = |
| 282 (base::subtle::Acquire_Load(&typing_detected_) != false); | 280 (base::subtle::Acquire_Load(&typing_detected_) != false); |
| 283 GetAecStats(audio_processing_.get(), stats); | 281 GetAecStats(audio_processing_.get(), stats); |
| 284 } | 282 } |
| 285 | 283 |
| 286 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( | 284 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( |
| 287 const blink::WebMediaConstraints& constraints, int effects, | 285 const blink::WebMediaConstraints& constraints, int effects) { |
| 288 MediaStreamType type) { | |
| 289 DCHECK(!audio_processing_); | 286 DCHECK(!audio_processing_); |
| 290 | 287 |
| 291 RTCMediaConstraints native_constraints(constraints); | 288 MediaAudioConstraints audio_constraints(constraints, effects); |
| 292 | 289 |
| 293 // Audio mirroring can be enabled even though audio processing is otherwise | 290 // Audio mirroring can be enabled even though audio processing is otherwise |
| 294 // disabled. | 291 // disabled. |
| 295 audio_mirroring_ = GetPropertyFromConstraints( | 292 audio_mirroring_ = audio_constraints.GetProperty( |
| 296 &native_constraints, webrtc::MediaConstraintsInterface::kAudioMirroring); | 293 MediaAudioConstraints::kGoogAudioMirroring); |
| 297 | 294 |
| 298 if (!IsAudioTrackProcessingEnabled()) { | 295 if (!IsAudioTrackProcessingEnabled()) { |
| 299 RecordProcessingState(AUDIO_PROCESSING_IN_WEBRTC); | 296 RecordProcessingState(AUDIO_PROCESSING_IN_WEBRTC); |
| 300 return; | 297 return; |
| 301 } | 298 } |
| 302 | 299 |
| 303 // Only apply the fixed constraints for gUM of MEDIA_DEVICE_AUDIO_CAPTURE. | 300 // |kEchoCancellation| is used as a master control on enabling/disabling |
| 304 DCHECK(IsAudioMediaType(type)); | 301 // the audio processing. |
| 305 if (type == MEDIA_DEVICE_AUDIO_CAPTURE) | 302 // If |kEchoCancellation| is specified in |audio_constraints|, it will use |
| 306 ApplyFixedAudioConstraints(&native_constraints); | 303 // the defined value there, otherwise use the default value defined by |
| 307 | 304 // kDefaultAudioConstraints in media_stream_audio_processor_options.cc. |
| 308 if (effects & media::AudioParameters::ECHO_CANCELLER) { | 305 const bool echo_cancellation = audio_constraints.GetProperty( |
| 309 // If platform echo canceller is enabled, disable the software AEC. | 306 MediaAudioConstraints::kEchoCancellation); |
| 310 native_constraints.AddMandatory( | 307 if (!echo_cancellation) { |
| 311 MediaConstraintsInterface::kEchoCancellation, | 308 RecordProcessingState(AUDIO_PROCESSING_DISABLED); |
| 312 MediaConstraintsInterface::kValueFalse, true); | 309 return; |
| 313 } | 310 } |
| 314 | 311 |
| 315 #if defined(OS_IOS) | 312 #if defined(OS_IOS) |
| 316 // On iOS, VPIO provides built-in AEC and AGC. | 313 // On iOS, VPIO provides built-in AEC and AGC. |
| 317 const bool enable_aec = false; | 314 const bool goog_aec = false; |
| 318 const bool enable_agc = false; | 315 const bool goog_agc = false; |
| 319 #else | 316 #else |
| 320 const bool enable_aec = GetPropertyFromConstraints( | 317 // TODO(xians): goog_aec should just be echo_cancellation. |
| 321 &native_constraints, MediaConstraintsInterface::kEchoCancellation); | 318 const bool goog_aec = audio_constraints.GetProperty( |
| 322 const bool enable_agc = GetPropertyFromConstraints( | 319 MediaAudioConstraints::kGoogEchoCancellation); |
| 323 &native_constraints, webrtc::MediaConstraintsInterface::kAutoGainControl); | 320 const bool goog_agc = audio_constraints.GetProperty( |
| 321 MediaAudioConstraints::kGoogAutoGainControl); |
| 324 #endif | 322 #endif |
| 325 | 323 |
| 326 #if defined(OS_IOS) || defined(OS_ANDROID) | 324 #if defined(OS_IOS) || defined(OS_ANDROID) |
| 327 const bool enable_experimental_aec = false; | 325 const bool goog_experimental_aec = false; |
| 328 const bool enable_typing_detection = false; | 326 const bool goog_typing_detection = false; |
| 329 #else | 327 #else |
| 330 const bool enable_experimental_aec = GetPropertyFromConstraints( | 328 const bool goog_experimental_aec = audio_constraints.GetProperty( |
| 331 &native_constraints, | 329 MediaAudioConstraints::kGoogExperimentalEchoCancellation); |
| 332 MediaConstraintsInterface::kExperimentalEchoCancellation); | 330 const bool goog_typing_detection = audio_constraints.GetProperty( |
| 333 const bool enable_typing_detection = GetPropertyFromConstraints( | 331 MediaAudioConstraints::kGoogTypingNoiseDetection); |
| 334 &native_constraints, MediaConstraintsInterface::kTypingNoiseDetection); | |
| 335 #endif | 332 #endif |
| 336 | 333 |
| 337 const bool enable_ns = GetPropertyFromConstraints( | 334 const bool goog_ns = audio_constraints.GetProperty( |
| 338 &native_constraints, MediaConstraintsInterface::kNoiseSuppression); | 335 MediaAudioConstraints::kGoogNoiseSuppression); |
| 339 const bool enable_experimental_ns = GetPropertyFromConstraints( | 336 const bool goog_experimental_ns = audio_constraints.GetProperty( |
| 340 &native_constraints, | 337 MediaAudioConstraints::kGoogExperimentalNoiseSuppression); |
| 341 MediaConstraintsInterface::kExperimentalNoiseSuppression); | 338 const bool goog_high_pass_filter = audio_constraints.GetProperty( |
| 342 const bool enable_high_pass_filter = GetPropertyFromConstraints( | 339 MediaAudioConstraints::kGoogHighpassFilter); |
| 343 &native_constraints, MediaConstraintsInterface::kHighpassFilter); | |
| 344 | 340 |
| 345 // Return immediately if no audio processing component is enabled. | 341 // Return immediately if no goog constraint is enabled. |
| 346 if (!enable_aec && !enable_experimental_aec && !enable_ns && | 342 if (!goog_aec && !goog_experimental_aec && !goog_ns && |
| 347 !enable_high_pass_filter && !enable_typing_detection && !enable_agc && | 343 !goog_high_pass_filter && !goog_typing_detection && |
| 348 !enable_experimental_ns) { | 344 !goog_agc && !goog_experimental_ns) { |
| 349 RecordProcessingState(AUDIO_PROCESSING_DISABLED); | 345 RecordProcessingState(AUDIO_PROCESSING_DISABLED); |
| 350 return; | 346 return; |
| 351 } | 347 } |
| 352 | 348 |
| 353 // Create and configure the webrtc::AudioProcessing. | 349 // Create and configure the webrtc::AudioProcessing. |
| 354 audio_processing_.reset(webrtc::AudioProcessing::Create(0)); | 350 audio_processing_.reset(webrtc::AudioProcessing::Create(0)); |
| 355 // TODO(ajm): Replace with AudioProcessing::Initialize() when this rolls to | 351 // TODO(ajm): Replace with AudioProcessing::Initialize() when this rolls to |
| 356 // Chromium: http://review.webrtc.org/9919004/ | 352 // Chromium: http://review.webrtc.org/9919004/ |
| 357 CHECK_EQ(0, | 353 CHECK_EQ(0, |
| 358 audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate)); | 354 audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate)); |
| 359 | 355 |
| 360 // Enable the audio processing components. | 356 // Enable the audio processing components. |
| 361 if (enable_aec) { | 357 if (goog_aec) { |
| 362 EnableEchoCancellation(audio_processing_.get()); | 358 EnableEchoCancellation(audio_processing_.get()); |
| 363 if (enable_experimental_aec) | 359 |
| 360 if (goog_experimental_aec) |
| 364 EnableExperimentalEchoCancellation(audio_processing_.get()); | 361 EnableExperimentalEchoCancellation(audio_processing_.get()); |
| 365 | 362 |
| 366 if (playout_data_source_) | 363 if (playout_data_source_) |
| 367 playout_data_source_->AddPlayoutSink(this); | 364 playout_data_source_->AddPlayoutSink(this); |
| 368 } | 365 } |
| 369 | 366 |
| 370 if (enable_ns) | 367 if (goog_ns) |
| 371 EnableNoiseSuppression(audio_processing_.get()); | 368 EnableNoiseSuppression(audio_processing_.get()); |
| 372 | 369 |
| 373 if (enable_experimental_ns) | 370 if (goog_experimental_ns) |
| 374 EnableExperimentalNoiseSuppression(audio_processing_.get()); | 371 EnableExperimentalNoiseSuppression(audio_processing_.get()); |
| 375 | 372 |
| 376 if (enable_high_pass_filter) | 373 if (goog_high_pass_filter) |
| 377 EnableHighPassFilter(audio_processing_.get()); | 374 EnableHighPassFilter(audio_processing_.get()); |
| 378 | 375 |
| 379 if (enable_typing_detection) { | 376 if (goog_typing_detection) { |
| 380 // TODO(xians): Remove this |typing_detector_| after the typing suppression | 377 // TODO(xians): Remove this |typing_detector_| after the typing suppression |
| 381 // is enabled by default. | 378 // is enabled by default. |
| 382 typing_detector_.reset(new webrtc::TypingDetection()); | 379 typing_detector_.reset(new webrtc::TypingDetection()); |
| 383 EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); | 380 EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); |
| 384 } | 381 } |
| 385 | 382 |
| 386 if (enable_agc) | 383 if (goog_agc) |
| 387 EnableAutomaticGainControl(audio_processing_.get()); | 384 EnableAutomaticGainControl(audio_processing_.get()); |
| 388 | 385 |
| 389 RecordProcessingState(AUDIO_PROCESSING_ENABLED); | 386 RecordProcessingState(AUDIO_PROCESSING_ENABLED); |
| 390 } | 387 } |
| 391 | 388 |
| 392 void MediaStreamAudioProcessor::InitializeCaptureConverter( | 389 void MediaStreamAudioProcessor::InitializeCaptureConverter( |
| 393 const media::AudioParameters& source_params) { | 390 const media::AudioParameters& source_params) { |
| 394 DCHECK(main_thread_checker_.CalledOnValidThread()); | 391 DCHECK(main_thread_checker_.CalledOnValidThread()); |
| 395 DCHECK(source_params.IsValid()); | 392 DCHECK(source_params.IsValid()); |
| 396 | 393 |
| (...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 509 | 506 |
| 510 StopAecDump(); | 507 StopAecDump(); |
| 511 | 508 |
| 512 if (playout_data_source_) | 509 if (playout_data_source_) |
| 513 playout_data_source_->RemovePlayoutSink(this); | 510 playout_data_source_->RemovePlayoutSink(this); |
| 514 | 511 |
| 515 audio_processing_.reset(); | 512 audio_processing_.reset(); |
| 516 } | 513 } |
| 517 | 514 |
| 518 } // namespace content | 515 } // namespace content |
| OLD | NEW |