OLD | NEW |
---|---|
1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media/media_stream_audio_processor.h" | 5 #include "content/renderer/media/media_stream_audio_processor.h" |
6 | 6 |
7 #include "base/command_line.h" | 7 #include "base/command_line.h" |
8 #include "base/debug/trace_event.h" | 8 #include "base/debug/trace_event.h" |
9 #include "content/public/common/content_switches.h" | 9 #include "content/public/common/content_switches.h" |
10 #include "content/renderer/media/media_stream_audio_processor_options.h" | 10 #include "content/renderer/media/media_stream_audio_processor_options.h" |
11 #include "content/renderer/media/rtc_media_constraints.h" | 11 #include "content/renderer/media/rtc_media_constraints.h" |
12 #include "media/audio/audio_parameters.h" | 12 #include "media/audio/audio_parameters.h" |
13 #include "media/base/audio_converter.h" | 13 #include "media/base/audio_converter.h" |
14 #include "media/base/audio_fifo.h" | 14 #include "media/base/audio_fifo.h" |
15 #include "media/base/channel_layout.h" | 15 #include "media/base/channel_layout.h" |
16 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" | 16 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" |
17 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface .h" | 17 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface .h" |
18 | 18 |
19 namespace content { | 19 namespace content { |
20 | 20 |
21 namespace { | 21 namespace { |
22 | 22 |
23 using webrtc::AudioProcessing; | 23 using webrtc::AudioProcessing; |
24 using webrtc::MediaConstraintsInterface; | 24 using webrtc::MediaConstraintsInterface; |
25 | 25 |
26 #if defined(ANDROID) | 26 #if defined(OS_ANDROID) |
27 const int kAudioProcessingSampleRate = 16000; | 27 const int kAudioProcessingSampleRate = 16000; |
28 #else | 28 #else |
29 const int kAudioProcessingSampleRate = 32000; | 29 const int kAudioProcessingSampleRate = 32000; |
30 #endif | 30 #endif |
31 const int kAudioProcessingNumberOfChannel = 1; | 31 const int kAudioProcessingNumberOfChannel = 1; |
32 | 32 |
33 const int kMaxNumberOfBuffersInFifo = 2; | 33 const int kMaxNumberOfBuffersInFifo = 2; |
34 | 34 |
35 } // namespace | 35 } // namespace |
36 | 36 |
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
135 // Handles mixing and resampling between input and output parameters. | 135 // Handles mixing and resampling between input and output parameters. |
136 media::AudioConverter audio_converter_; | 136 media::AudioConverter audio_converter_; |
137 scoped_ptr<media::AudioBus> audio_wrapper_; | 137 scoped_ptr<media::AudioBus> audio_wrapper_; |
138 scoped_ptr<media::AudioFifo> fifo_; | 138 scoped_ptr<media::AudioFifo> fifo_; |
139 }; | 139 }; |
140 | 140 |
141 MediaStreamAudioProcessor::MediaStreamAudioProcessor( | 141 MediaStreamAudioProcessor::MediaStreamAudioProcessor( |
142 const media::AudioParameters& source_params, | 142 const media::AudioParameters& source_params, |
143 const blink::WebMediaConstraints& constraints, | 143 const blink::WebMediaConstraints& constraints, |
144 int effects) | 144 int effects) |
145 : render_delay_ms_(0) { | 145 : render_delay_ms_(0), |
146 audio_mirroring_(false) { | |
146 capture_thread_checker_.DetachFromThread(); | 147 capture_thread_checker_.DetachFromThread(); |
147 render_thread_checker_.DetachFromThread(); | 148 render_thread_checker_.DetachFromThread(); |
148 InitializeAudioProcessingModule(constraints, effects); | 149 InitializeAudioProcessingModule(constraints, effects); |
149 InitializeCaptureConverter(source_params); | 150 InitializeCaptureConverter(source_params); |
150 } | 151 } |
151 | 152 |
152 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { | 153 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { |
153 DCHECK(main_thread_checker_.CalledOnValidThread()); | 154 DCHECK(main_thread_checker_.CalledOnValidThread()); |
154 StopAudioProcessing(); | 155 StopAudioProcessing(); |
155 } | 156 } |
(...skipping 28 matching lines...) Expand all Loading... | |
184 render_data_bus_->FromInterleaved(render_audio, | 185 render_data_bus_->FromInterleaved(render_audio, |
185 render_data_bus_->frames(), | 186 render_data_bus_->frames(), |
186 sizeof(render_audio[0])); | 187 sizeof(render_audio[0])); |
187 render_converter_->Push(render_data_bus_.get()); | 188 render_converter_->Push(render_data_bus_.get()); |
188 while (render_converter_->Convert(&render_frame_)) | 189 while (render_converter_->Convert(&render_frame_)) |
189 audio_processing_->AnalyzeReverseStream(&render_frame_); | 190 audio_processing_->AnalyzeReverseStream(&render_frame_); |
190 } | 191 } |
191 | 192 |
192 bool MediaStreamAudioProcessor::ProcessAndConsumeData( | 193 bool MediaStreamAudioProcessor::ProcessAndConsumeData( |
193 base::TimeDelta capture_delay, int volume, bool key_pressed, | 194 base::TimeDelta capture_delay, int volume, bool key_pressed, |
194 int16** out) { | 195 int* new_volume, int16** out) { |
195 DCHECK(capture_thread_checker_.CalledOnValidThread()); | 196 DCHECK(capture_thread_checker_.CalledOnValidThread()); |
196 TRACE_EVENT0("audio", | 197 TRACE_EVENT0("audio", |
197 "MediaStreamAudioProcessor::ProcessAndConsumeData"); | 198 "MediaStreamAudioProcessor::ProcessAndConsumeData"); |
199 *new_volume = 0; | |
tommi (sloooow) - chröme
2014/01/24 12:41:22
Is it correct to do this here and not below the Co
no longer working on chromium
2014/01/24 12:46:53
Right, actually the new_volume won't be used if th
| |
198 | 200 |
199 if (!capture_converter_->Convert(&capture_frame_)) | 201 if (!capture_converter_->Convert(&capture_frame_)) |
200 return false; | 202 return false; |
201 | 203 |
202 ProcessData(&capture_frame_, capture_delay, volume, key_pressed); | 204 *new_volume = ProcessData(&capture_frame_, capture_delay, volume, |
205 key_pressed); | |
203 *out = capture_frame_.data_; | 206 *out = capture_frame_.data_; |
204 | 207 |
205 return true; | 208 return true; |
206 } | 209 } |
207 | 210 |
208 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { | 211 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { |
209 return capture_converter_->source_parameters(); | 212 return capture_converter_->source_parameters(); |
210 } | 213 } |
211 | 214 |
212 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { | 215 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { |
213 return capture_converter_->sink_parameters(); | 216 return capture_converter_->sink_parameters(); |
214 } | 217 } |
215 | 218 |
216 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( | 219 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( |
217 const blink::WebMediaConstraints& constraints, int effects) { | 220 const blink::WebMediaConstraints& constraints, int effects) { |
218 DCHECK(!audio_processing_); | 221 DCHECK(!audio_processing_); |
219 if (!CommandLine::ForCurrentProcess()->HasSwitch( | 222 if (!CommandLine::ForCurrentProcess()->HasSwitch( |
220 switches::kEnableAudioTrackProcessing)) { | 223 switches::kEnableAudioTrackProcessing)) { |
221 return; | 224 return; |
222 } | 225 } |
223 | 226 |
224 RTCMediaConstraints native_constraints(constraints); | 227 RTCMediaConstraints native_constraints(constraints); |
225 ApplyFixedAudioConstraints(&native_constraints); | 228 ApplyFixedAudioConstraints(&native_constraints); |
226 if (effects & media::AudioParameters::ECHO_CANCELLER) { | 229 if (effects & media::AudioParameters::ECHO_CANCELLER) { |
227 // If platform echo cancellator is enabled, disable the software AEC. | 230 // If platform echo canceller is enabled, disable the software AEC. |
228 native_constraints.AddMandatory( | 231 native_constraints.AddMandatory( |
229 MediaConstraintsInterface::kEchoCancellation, | 232 MediaConstraintsInterface::kEchoCancellation, |
230 MediaConstraintsInterface::kValueFalse, true); | 233 MediaConstraintsInterface::kValueFalse, true); |
231 } | 234 } |
232 | 235 |
236 #if defined(OS_IOS) | |
237 // On iOS, VPIO provides built-in AEC and AGC. | |
238 const bool enable_aec = false; | |
239 const bool enable_agc = false; | |
240 #else | |
233 const bool enable_aec = GetPropertyFromConstraints( | 241 const bool enable_aec = GetPropertyFromConstraints( |
234 &native_constraints, MediaConstraintsInterface::kEchoCancellation); | 242 &native_constraints, MediaConstraintsInterface::kEchoCancellation); |
235 const bool enable_ns = GetPropertyFromConstraints( | 243 const bool enable_agc = GetPropertyFromConstraints( |
236 &native_constraints, MediaConstraintsInterface::kNoiseSuppression); | 244 &native_constraints, webrtc::MediaConstraintsInterface::kAutoGainControl); |
237 const bool enable_high_pass_filter = GetPropertyFromConstraints( | 245 #endif |
238 &native_constraints, MediaConstraintsInterface::kHighpassFilter); | 246 |
239 #if defined(IOS) || defined(ANDROID) | 247 #if defined(OS_IOS) || defined(OS_ANDROID) |
240 const bool enable_experimental_aec = false; | 248 const bool enable_experimental_aec = false; |
241 const bool enable_typing_detection = false; | 249 const bool enable_typing_detection = false; |
242 #else | 250 #else |
243 const bool enable_experimental_aec = GetPropertyFromConstraints( | 251 const bool enable_experimental_aec = GetPropertyFromConstraints( |
244 &native_constraints, | 252 &native_constraints, |
245 MediaConstraintsInterface::kExperimentalEchoCancellation); | 253 MediaConstraintsInterface::kExperimentalEchoCancellation); |
246 const bool enable_typing_detection = GetPropertyFromConstraints( | 254 const bool enable_typing_detection = GetPropertyFromConstraints( |
247 &native_constraints, MediaConstraintsInterface::kTypingNoiseDetection); | 255 &native_constraints, MediaConstraintsInterface::kTypingNoiseDetection); |
248 #endif | 256 #endif |
249 | 257 |
258 const bool enable_ns = GetPropertyFromConstraints( | |
259 &native_constraints, MediaConstraintsInterface::kNoiseSuppression); | |
260 const bool enable_high_pass_filter = GetPropertyFromConstraints( | |
261 &native_constraints, MediaConstraintsInterface::kHighpassFilter); | |
262 | |
263 audio_mirroring_ = GetPropertyFromConstraints( | |
264 &native_constraints, webrtc::MediaConstraintsInterface::kAudioMirroring); | |
265 | |
250 // Return immediately if no audio processing component is enabled. | 266 // Return immediately if no audio processing component is enabled. |
251 if (!enable_aec && !enable_experimental_aec && !enable_ns && | 267 if (!enable_aec && !enable_experimental_aec && !enable_ns && |
252 !enable_high_pass_filter && !enable_typing_detection) { | 268 !enable_high_pass_filter && !enable_typing_detection && !enable_agc) { |
253 return; | 269 return; |
254 } | 270 } |
255 | 271 |
256 // Create and configure the webrtc::AudioProcessing. | 272 // Create and configure the webrtc::AudioProcessing. |
257 audio_processing_.reset(webrtc::AudioProcessing::Create(0)); | 273 audio_processing_.reset(webrtc::AudioProcessing::Create(0)); |
258 | 274 |
259 // Enable the audio processing components. | 275 // Enable the audio processing components. |
260 if (enable_aec) { | 276 if (enable_aec) { |
261 EnableEchoCancellation(audio_processing_.get()); | 277 EnableEchoCancellation(audio_processing_.get()); |
262 if (enable_experimental_aec) | 278 if (enable_experimental_aec) |
263 EnableExperimentalEchoCancellation(audio_processing_.get()); | 279 EnableExperimentalEchoCancellation(audio_processing_.get()); |
264 } | 280 } |
265 | 281 |
266 if (enable_ns) | 282 if (enable_ns) |
267 EnableNoiseSuppression(audio_processing_.get()); | 283 EnableNoiseSuppression(audio_processing_.get()); |
268 | 284 |
269 if (enable_high_pass_filter) | 285 if (enable_high_pass_filter) |
270 EnableHighPassFilter(audio_processing_.get()); | 286 EnableHighPassFilter(audio_processing_.get()); |
271 | 287 |
272 if (enable_typing_detection) | 288 if (enable_typing_detection) |
273 EnableTypingDetection(audio_processing_.get()); | 289 EnableTypingDetection(audio_processing_.get()); |
274 | 290 |
291 if (enable_agc) | |
292 EnableAutomaticGainControl(audio_processing_.get()); | |
275 | 293 |
276 // Configure the audio format the audio processing is running on. This | 294 // Configure the audio format the audio processing is running on. This |
277 // has to be done after all the needed components are enabled. | 295 // has to be done after all the needed components are enabled. |
278 CHECK_EQ(audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate), | 296 CHECK_EQ(audio_processing_->set_sample_rate_hz(kAudioProcessingSampleRate), |
279 0); | 297 0); |
280 CHECK_EQ(audio_processing_->set_num_channels(kAudioProcessingNumberOfChannel, | 298 CHECK_EQ(audio_processing_->set_num_channels(kAudioProcessingNumberOfChannel, |
281 kAudioProcessingNumberOfChannel), | 299 kAudioProcessingNumberOfChannel), |
282 0); | 300 0); |
283 } | 301 } |
284 | 302 |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
334 media::AudioParameters sink_params( | 352 media::AudioParameters sink_params( |
335 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, | 353 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, |
336 media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16, | 354 media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16, |
337 kAudioProcessingSampleRate / 100); | 355 kAudioProcessingSampleRate / 100); |
338 render_converter_.reset( | 356 render_converter_.reset( |
339 new MediaStreamAudioConverter(source_params, sink_params)); | 357 new MediaStreamAudioConverter(source_params, sink_params)); |
340 render_data_bus_ = media::AudioBus::Create(number_of_channels, | 358 render_data_bus_ = media::AudioBus::Create(number_of_channels, |
341 frames_per_buffer); | 359 frames_per_buffer); |
342 } | 360 } |
343 | 361 |
344 void MediaStreamAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame, | 362 int MediaStreamAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame, |
345 base::TimeDelta capture_delay, | 363 base::TimeDelta capture_delay, |
346 int volume, | 364 int volume, |
347 bool key_pressed) { | 365 bool key_pressed) { |
348 DCHECK(capture_thread_checker_.CalledOnValidThread()); | 366 DCHECK(capture_thread_checker_.CalledOnValidThread()); |
349 if (!audio_processing_) | 367 if (!audio_processing_) |
350 return; | 368 return 0; |
351 | 369 |
352 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::Process10MsData"); | 370 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessData"); |
353 DCHECK_EQ(audio_processing_->sample_rate_hz(), | 371 DCHECK_EQ(audio_processing_->sample_rate_hz(), |
354 capture_converter_->sink_parameters().sample_rate()); | 372 capture_converter_->sink_parameters().sample_rate()); |
355 DCHECK_EQ(audio_processing_->num_input_channels(), | 373 DCHECK_EQ(audio_processing_->num_input_channels(), |
356 capture_converter_->sink_parameters().channels()); | 374 capture_converter_->sink_parameters().channels()); |
357 DCHECK_EQ(audio_processing_->num_output_channels(), | 375 DCHECK_EQ(audio_processing_->num_output_channels(), |
358 capture_converter_->sink_parameters().channels()); | 376 capture_converter_->sink_parameters().channels()); |
359 | 377 |
360 base::subtle::Atomic32 render_delay_ms = | 378 base::subtle::Atomic32 render_delay_ms = |
361 base::subtle::Acquire_Load(&render_delay_ms_); | 379 base::subtle::Acquire_Load(&render_delay_ms_); |
362 int64 capture_delay_ms = capture_delay.InMilliseconds(); | 380 int64 capture_delay_ms = capture_delay.InMilliseconds(); |
363 DCHECK_LT(capture_delay_ms, | 381 DCHECK_LT(capture_delay_ms, |
364 std::numeric_limits<base::subtle::Atomic32>::max()); | 382 std::numeric_limits<base::subtle::Atomic32>::max()); |
365 int total_delay_ms = capture_delay_ms + render_delay_ms; | 383 int total_delay_ms = capture_delay_ms + render_delay_ms; |
366 if (total_delay_ms > 1000) { | 384 if (total_delay_ms > 300) { |
367 LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms | 385 LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms |
368 << "ms; render delay: " << render_delay_ms << "ms"; | 386 << "ms; render delay: " << render_delay_ms << "ms"; |
369 } | 387 } |
370 | 388 |
371 audio_processing_->set_stream_delay_ms(total_delay_ms); | 389 audio_processing_->set_stream_delay_ms(total_delay_ms); |
372 webrtc::GainControl* agc = audio_processing_->gain_control(); | 390 webrtc::GainControl* agc = audio_processing_->gain_control(); |
373 int err = agc->set_stream_analog_level(volume); | 391 int err = agc->set_stream_analog_level(volume); |
374 DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; | 392 DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; |
375 err = audio_processing_->ProcessStream(audio_frame); | 393 err = audio_processing_->ProcessStream(audio_frame); |
376 DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; | 394 DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; |
377 | 395 |
378 // TODO(xians): Add support for AGC, typing detection, audio level | 396 // TODO(xians): Add support for typing detection, audio level calculation. |
379 // calculation, stereo swapping. | 397 |
398 if (audio_mirroring_ && audio_frame->num_channels_ == 2) { | |
399 // TODO(xians): Swap the stereo channels after switching to media::AudioBus. | |
400 } | |
401 | |
402 // Return 0 if the volume has not been changed, otherwise return the new | |
403 // volume. | |
404 return (agc->stream_analog_level() == volume) ? | |
405 0 : agc->stream_analog_level(); | |
380 } | 406 } |
381 | 407 |
382 void MediaStreamAudioProcessor::StopAudioProcessing() { | 408 void MediaStreamAudioProcessor::StopAudioProcessing() { |
383 if (!audio_processing_.get()) | 409 if (!audio_processing_.get()) |
384 return; | 410 return; |
385 | 411 |
386 audio_processing_.reset(); | 412 audio_processing_.reset(); |
387 } | 413 } |
388 | 414 |
389 } // namespace content | 415 } // namespace content |
OLD | NEW |