OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" | 11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" |
12 | 12 |
13 #include <assert.h> | 13 #include <assert.h> |
14 #include <algorithm> | |
14 | 15 |
15 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
16 #include "webrtc/base/platform_file.h" | 17 #include "webrtc/base/platform_file.h" |
17 #include "webrtc/common_audio/include/audio_util.h" | 18 #include "webrtc/common_audio/include/audio_util.h" |
18 #include "webrtc/common_audio/channel_buffer.h" | 19 #include "webrtc/common_audio/channel_buffer.h" |
19 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar y.h" | 20 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar y.h" |
20 extern "C" { | 21 extern "C" { |
21 #include "webrtc/modules/audio_processing/aec/aec_core.h" | 22 #include "webrtc/modules/audio_processing/aec/aec_core.h" |
22 } | 23 } |
23 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" | 24 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" |
(...skipping 26 matching lines...) Expand all Loading... | |
50 | 51 |
51 #define RETURN_ON_ERR(expr) \ | 52 #define RETURN_ON_ERR(expr) \ |
52 do { \ | 53 do { \ |
53 int err = (expr); \ | 54 int err = (expr); \ |
54 if (err != kNoError) { \ | 55 if (err != kNoError) { \ |
55 return err; \ | 56 return err; \ |
56 } \ | 57 } \ |
57 } while (0) | 58 } while (0) |
58 | 59 |
59 namespace webrtc { | 60 namespace webrtc { |
61 namespace { | |
62 | |
63 static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { | |
64 switch (layout) { | |
65 case AudioProcessing::kMono: | |
66 case AudioProcessing::kStereo: | |
67 return false; | |
68 case AudioProcessing::kMonoAndKeyboard: | |
69 case AudioProcessing::kStereoAndKeyboard: | |
70 return true; | |
71 } | |
72 | |
73 assert(false); | |
74 return false; | |
75 } | |
76 | |
77 } // namespace | |
60 | 78 |
61 // Throughout webrtc, it's assumed that success is represented by zero. | 79 // Throughout webrtc, it's assumed that success is represented by zero. |
62 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); | 80 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
63 | 81 |
64 // This class has two main functionalities: | 82 // This class has two main functionalities: |
65 // | 83 // |
66 // 1) It is returned instead of the real GainControl after the new AGC has been | 84 // 1) It is returned instead of the real GainControl after the new AGC has been |
67 // enabled in order to prevent an outside user from overriding compression | 85 // enabled in order to prevent an outside user from overriding compression |
68 // settings. It doesn't do anything in its implementation, except for | 86 // settings. It doesn't do anything in its implementation, except for |
69 // delegating the const methods and Enable calls to the real GainControl, so | 87 // delegating the const methods and Enable calls to the real GainControl, so |
70 // AGC can still be disabled. | 88 // AGC can still be disabled. |
71 // | 89 // |
72 // 2) It is injected into AgcManagerDirect and implements volume callbacks for | 90 // 2) It is injected into AgcManagerDirect and implements volume callbacks for |
73 // getting and setting the volume level. It just caches this value to be used | 91 // getting and setting the volume level. It just caches this value to be used |
74 // in VoiceEngine later. | 92 // in VoiceEngine later. |
75 class GainControlForNewAgc : public GainControl, public VolumeCallbacks { | 93 class GainControlForNewAgc : public GainControl, public VolumeCallbacks { |
76 public: | 94 public: |
77 explicit GainControlForNewAgc(GainControlImpl* gain_control) | 95 explicit GainControlForNewAgc(GainControlImpl* gain_control) |
78 : real_gain_control_(gain_control), | 96 : real_gain_control_(gain_control), volume_(0) {} |
79 volume_(0) { | |
80 } | |
81 | 97 |
82 // GainControl implementation. | 98 // GainControl implementation. |
83 int Enable(bool enable) override { | 99 int Enable(bool enable) override { |
84 return real_gain_control_->Enable(enable); | 100 return real_gain_control_->Enable(enable); |
85 } | 101 } |
86 bool is_enabled() const override { return real_gain_control_->is_enabled(); } | 102 bool is_enabled() const override { return real_gain_control_->is_enabled(); } |
87 int set_stream_analog_level(int level) override { | 103 int set_stream_analog_level(int level) override { |
88 volume_ = level; | 104 volume_ = level; |
89 return AudioProcessing::kNoError; | 105 return AudioProcessing::kNoError; |
90 } | 106 } |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
159 gain_control_(NULL), | 175 gain_control_(NULL), |
160 high_pass_filter_(NULL), | 176 high_pass_filter_(NULL), |
161 level_estimator_(NULL), | 177 level_estimator_(NULL), |
162 noise_suppression_(NULL), | 178 noise_suppression_(NULL), |
163 voice_detection_(NULL), | 179 voice_detection_(NULL), |
164 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 180 crit_(CriticalSectionWrapper::CreateCriticalSection()), |
165 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 181 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
166 debug_file_(FileWrapper::Create()), | 182 debug_file_(FileWrapper::Create()), |
167 event_msg_(new audioproc::Event()), | 183 event_msg_(new audioproc::Event()), |
168 #endif | 184 #endif |
169 fwd_in_format_(kSampleRate16kHz, 1), | 185 api_format_({{{kSampleRate16kHz, 1, false}, |
186 {kSampleRate16kHz, 1, false}, | |
187 {kSampleRate16kHz, 1, false}}}), | |
170 fwd_proc_format_(kSampleRate16kHz), | 188 fwd_proc_format_(kSampleRate16kHz), |
171 fwd_out_format_(kSampleRate16kHz, 1), | |
172 rev_in_format_(kSampleRate16kHz, 1), | |
173 rev_proc_format_(kSampleRate16kHz, 1), | 189 rev_proc_format_(kSampleRate16kHz, 1), |
174 split_rate_(kSampleRate16kHz), | 190 split_rate_(kSampleRate16kHz), |
175 stream_delay_ms_(0), | 191 stream_delay_ms_(0), |
176 delay_offset_ms_(0), | 192 delay_offset_ms_(0), |
177 was_stream_delay_set_(false), | 193 was_stream_delay_set_(false), |
178 last_stream_delay_ms_(0), | 194 last_stream_delay_ms_(0), |
179 last_aec_system_delay_ms_(0), | 195 last_aec_system_delay_ms_(0), |
180 stream_delay_jumps_(-1), | 196 stream_delay_jumps_(-1), |
181 aec_system_delay_jumps_(-1), | 197 aec_system_delay_jumps_(-1), |
182 output_will_be_muted_(false), | 198 output_will_be_muted_(false), |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
246 crit_ = NULL; | 262 crit_ = NULL; |
247 } | 263 } |
248 | 264 |
249 int AudioProcessingImpl::Initialize() { | 265 int AudioProcessingImpl::Initialize() { |
250 CriticalSectionScoped crit_scoped(crit_); | 266 CriticalSectionScoped crit_scoped(crit_); |
251 return InitializeLocked(); | 267 return InitializeLocked(); |
252 } | 268 } |
253 | 269 |
254 int AudioProcessingImpl::set_sample_rate_hz(int rate) { | 270 int AudioProcessingImpl::set_sample_rate_hz(int rate) { |
255 CriticalSectionScoped crit_scoped(crit_); | 271 CriticalSectionScoped crit_scoped(crit_); |
256 return InitializeLocked(rate, | 272 |
257 rate, | 273 ProcessingConfig processing_config = api_format_; |
258 rev_in_format_.rate(), | 274 processing_config.input_stream().set_sample_rate_hz(rate); |
259 fwd_in_format_.num_channels(), | 275 processing_config.output_stream().set_sample_rate_hz(rate); |
260 fwd_out_format_.num_channels(), | 276 return InitializeLocked(processing_config); |
261 rev_in_format_.num_channels()); | |
262 } | 277 } |
263 | 278 |
264 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 279 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
aluebs-webrtc
2015/07/14 23:12:43
Will this interface be replaced by one with a Proc
mgraczyk
2015/07/15 01:12:46
Yes, eventually. It'll be easiest to first commit
aluebs-webrtc
2015/07/15 18:04:06
Agreed. Just making sure that that is the plan.
| |
265 int output_sample_rate_hz, | 280 int output_sample_rate_hz, |
266 int reverse_sample_rate_hz, | 281 int reverse_sample_rate_hz, |
267 ChannelLayout input_layout, | 282 ChannelLayout input_layout, |
268 ChannelLayout output_layout, | 283 ChannelLayout output_layout, |
269 ChannelLayout reverse_layout) { | 284 ChannelLayout reverse_layout) { |
285 const ProcessingConfig processing_config = { | |
286 {{input_sample_rate_hz, ChannelsFromLayout(input_layout), | |
287 LayoutHasKeyboard(input_layout)}, | |
288 {output_sample_rate_hz, ChannelsFromLayout(output_layout), | |
289 LayoutHasKeyboard(output_layout)}, | |
290 {reverse_sample_rate_hz, ChannelsFromLayout(reverse_layout), | |
291 LayoutHasKeyboard(reverse_layout)}}}; | |
292 | |
293 return Initialize(processing_config); | |
294 } | |
295 | |
296 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | |
270 CriticalSectionScoped crit_scoped(crit_); | 297 CriticalSectionScoped crit_scoped(crit_); |
271 return InitializeLocked(input_sample_rate_hz, | 298 return InitializeLocked(processing_config); |
272 output_sample_rate_hz, | |
273 reverse_sample_rate_hz, | |
274 ChannelsFromLayout(input_layout), | |
275 ChannelsFromLayout(output_layout), | |
276 ChannelsFromLayout(reverse_layout)); | |
277 } | 299 } |
278 | 300 |
279 int AudioProcessingImpl::InitializeLocked() { | 301 int AudioProcessingImpl::InitializeLocked() { |
280 const int fwd_audio_buffer_channels = beamformer_enabled_ ? | 302 const int fwd_audio_buffer_channels = |
281 fwd_in_format_.num_channels() : | 303 beamformer_enabled_ ? api_format_.input_stream().num_channels() |
282 fwd_out_format_.num_channels(); | 304 : api_format_.output_stream().num_channels(); |
283 render_audio_.reset(new AudioBuffer(rev_in_format_.samples_per_channel(), | 305 if (api_format_.reverse_stream().num_channels() > 0) { |
284 rev_in_format_.num_channels(), | 306 render_audio_.reset(new AudioBuffer( |
285 rev_proc_format_.samples_per_channel(), | 307 api_format_.reverse_stream().samples_per_channel(), |
286 rev_proc_format_.num_channels(), | 308 api_format_.reverse_stream().num_channels(), |
287 rev_proc_format_.samples_per_channel())); | 309 rev_proc_format_.samples_per_channel(), rev_proc_format_.num_channels(), |
288 capture_audio_.reset(new AudioBuffer(fwd_in_format_.samples_per_channel(), | 310 rev_proc_format_.samples_per_channel())); |
289 fwd_in_format_.num_channels(), | 311 } else { |
290 fwd_proc_format_.samples_per_channel(), | 312 render_audio_.reset(nullptr); |
aluebs-webrtc
2015/07/14 23:12:43
When do we want this case?
mgraczyk
2015/07/15 01:12:46
This is just saying "When there is no reverse stre
aluebs-webrtc
2015/07/15 18:04:06
Ack.
| |
291 fwd_audio_buffer_channels, | 313 } |
292 fwd_out_format_.samples_per_channel())); | 314 capture_audio_.reset(new AudioBuffer( |
315 api_format_.input_stream().samples_per_channel(), | |
316 api_format_.input_stream().num_channels(), | |
317 fwd_proc_format_.samples_per_channel(), fwd_audio_buffer_channels, | |
318 api_format_.output_stream().samples_per_channel())); | |
293 | 319 |
294 // Initialize all components. | 320 // Initialize all components. |
295 for (auto item : component_list_) { | 321 for (auto item : component_list_) { |
296 int err = item->Initialize(); | 322 int err = item->Initialize(); |
297 if (err != kNoError) { | 323 if (err != kNoError) { |
298 return err; | 324 return err; |
299 } | 325 } |
300 } | 326 } |
301 | 327 |
302 InitializeExperimentalAgc(); | 328 InitializeExperimentalAgc(); |
303 | 329 |
304 InitializeTransient(); | 330 InitializeTransient(); |
305 | 331 |
306 InitializeBeamformer(); | 332 InitializeBeamformer(); |
307 | 333 |
308 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 334 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
309 if (debug_file_->Open()) { | 335 if (debug_file_->Open()) { |
310 int err = WriteInitMessage(); | 336 int err = WriteInitMessage(); |
311 if (err != kNoError) { | 337 if (err != kNoError) { |
312 return err; | 338 return err; |
313 } | 339 } |
314 } | 340 } |
315 #endif | 341 #endif |
316 | 342 |
317 return kNoError; | 343 return kNoError; |
318 } | 344 } |
319 | 345 |
320 int AudioProcessingImpl::InitializeLocked(int input_sample_rate_hz, | 346 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { |
321 int output_sample_rate_hz, | 347 for (const auto& stream : config.streams) { |
322 int reverse_sample_rate_hz, | 348 if (stream.sample_rate_hz() < 0) { |
aluebs-webrtc
2015/07/14 23:12:43
<=
mgraczyk
2015/07/15 01:12:46
The reverse stream can have zero sampling rate and
aluebs-webrtc
2015/07/15 18:04:06
But here it looks like all streams can (although y
mgraczyk
2015/07/15 20:03:19
Yeah num_in_channels is checked to be nonzero belo
aluebs-webrtc
2015/07/15 21:29:17
Yes, but what about the sample rate?
mgraczyk
2015/07/15 21:53:56
I changed the check so that sample_rate only matte
| |
323 int num_input_channels, | 349 return kBadSampleRateError; |
324 int num_output_channels, | 350 } |
325 int num_reverse_channels) { | 351 if (stream.num_channels() < 0) { |
aluebs-webrtc
2015/07/14 23:12:43
< 1
mgraczyk
2015/07/15 01:12:46
See above
| |
326 if (input_sample_rate_hz <= 0 || | 352 return kBadNumberChannelsError; |
327 output_sample_rate_hz <= 0 || | 353 } |
328 reverse_sample_rate_hz <= 0) { | |
329 return kBadSampleRateError; | |
330 } | 354 } |
331 if (num_output_channels > num_input_channels) { | 355 |
332 return kBadNumberChannelsError; | 356 const int num_in_channels = config.input_stream().num_channels(); |
333 } | 357 const int num_out_channels = config.output_stream().num_channels(); |
334 // Only mono and stereo supported currently. | 358 |
335 if (num_input_channels > 2 || num_input_channels < 1 || | 359 // Need at least one input channel. |
336 num_output_channels > 2 || num_output_channels < 1 || | 360 // Need either one output channel or as many outputs as there are inputs. |
337 num_reverse_channels > 2 || num_reverse_channels < 1) { | 361 if (num_in_channels == 0 || |
338 return kBadNumberChannelsError; | 362 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { |
339 } | |
340 if (beamformer_enabled_ && | |
341 (static_cast<size_t>(num_input_channels) != array_geometry_.size() || | |
342 num_output_channels > 1)) { | |
343 return kBadNumberChannelsError; | 363 return kBadNumberChannelsError; |
344 } | 364 } |
345 | 365 |
346 fwd_in_format_.set(input_sample_rate_hz, num_input_channels); | 366 if (beamformer_enabled_ && |
347 fwd_out_format_.set(output_sample_rate_hz, num_output_channels); | 367 (static_cast<size_t>(config.input_stream().num_channels()) != |
aluebs-webrtc
2015/07/14 23:12:43
You can use num_in_channels here.
mgraczyk
2015/07/15 01:12:46
Done.
| |
348 rev_in_format_.set(reverse_sample_rate_hz, num_reverse_channels); | 368 array_geometry_.size() || num_out_channels > 1)) { |
369 return kBadNumberChannelsError; | |
370 } | |
371 | |
372 api_format_ = config; | |
349 | 373 |
350 // We process at the closest native rate >= min(input rate, output rate)... | 374 // We process at the closest native rate >= min(input rate, output rate)... |
351 int min_proc_rate = std::min(fwd_in_format_.rate(), fwd_out_format_.rate()); | 375 const int min_proc_rate = |
376 std::min(api_format_.input_stream().sample_rate_hz(), | |
377 api_format_.output_stream().sample_rate_hz()); | |
352 int fwd_proc_rate; | 378 int fwd_proc_rate; |
353 if (supports_48kHz_ && min_proc_rate > kSampleRate32kHz) { | 379 if (supports_48kHz_ && min_proc_rate > kSampleRate32kHz) { |
354 fwd_proc_rate = kSampleRate48kHz; | 380 fwd_proc_rate = kSampleRate48kHz; |
355 } else if (min_proc_rate > kSampleRate16kHz) { | 381 } else if (min_proc_rate > kSampleRate16kHz) { |
356 fwd_proc_rate = kSampleRate32kHz; | 382 fwd_proc_rate = kSampleRate32kHz; |
357 } else if (min_proc_rate > kSampleRate8kHz) { | 383 } else if (min_proc_rate > kSampleRate8kHz) { |
358 fwd_proc_rate = kSampleRate16kHz; | 384 fwd_proc_rate = kSampleRate16kHz; |
359 } else { | 385 } else { |
360 fwd_proc_rate = kSampleRate8kHz; | 386 fwd_proc_rate = kSampleRate8kHz; |
361 } | 387 } |
362 // ...with one exception. | 388 // ...with one exception. |
363 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) { | 389 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) { |
364 fwd_proc_rate = kSampleRate16kHz; | 390 fwd_proc_rate = kSampleRate16kHz; |
365 } | 391 } |
366 | 392 |
367 fwd_proc_format_.set(fwd_proc_rate); | 393 fwd_proc_format_ = StreamConfig(fwd_proc_rate); |
368 | 394 |
369 // We normally process the reverse stream at 16 kHz. Unless... | 395 // We normally process the reverse stream at 16 kHz. Unless... |
370 int rev_proc_rate = kSampleRate16kHz; | 396 int rev_proc_rate = kSampleRate16kHz; |
371 if (fwd_proc_format_.rate() == kSampleRate8kHz) { | 397 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { |
372 // ...the forward stream is at 8 kHz. | 398 // ...the forward stream is at 8 kHz. |
373 rev_proc_rate = kSampleRate8kHz; | 399 rev_proc_rate = kSampleRate8kHz; |
374 } else { | 400 } else { |
375 if (rev_in_format_.rate() == kSampleRate32kHz) { | 401 if (api_format_.reverse_stream().sample_rate_hz() == kSampleRate32kHz) { |
376 // ...or the input is at 32 kHz, in which case we use the splitting | 402 // ...or the input is at 32 kHz, in which case we use the splitting |
377 // filter rather than the resampler. | 403 // filter rather than the resampler. |
378 rev_proc_rate = kSampleRate32kHz; | 404 rev_proc_rate = kSampleRate32kHz; |
379 } | 405 } |
380 } | 406 } |
381 | 407 |
382 // Always downmix the reverse stream to mono for analysis. This has been | 408 // Always downmix the reverse stream to mono for analysis. This has been |
383 // demonstrated to work well for AEC in most practical scenarios. | 409 // demonstrated to work well for AEC in most practical scenarios. |
384 rev_proc_format_.set(rev_proc_rate, 1); | 410 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); |
385 | 411 |
386 if (fwd_proc_format_.rate() == kSampleRate32kHz || | 412 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
387 fwd_proc_format_.rate() == kSampleRate48kHz) { | 413 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { |
388 split_rate_ = kSampleRate16kHz; | 414 split_rate_ = kSampleRate16kHz; |
389 } else { | 415 } else { |
390 split_rate_ = fwd_proc_format_.rate(); | 416 split_rate_ = fwd_proc_format_.sample_rate_hz(); |
391 } | 417 } |
392 | 418 |
393 return InitializeLocked(); | 419 return InitializeLocked(); |
394 } | 420 } |
395 | 421 |
396 // Calls InitializeLocked() if any of the audio parameters have changed from | 422 // Calls InitializeLocked() if any of the audio parameters have changed from |
397 // their current values. | 423 // their current values. |
398 int AudioProcessingImpl::MaybeInitializeLocked(int input_sample_rate_hz, | 424 int AudioProcessingImpl::MaybeInitializeLocked( |
399 int output_sample_rate_hz, | 425 const ProcessingConfig& processing_config) { |
400 int reverse_sample_rate_hz, | 426 if (processing_config == api_format_) { |
401 int num_input_channels, | |
402 int num_output_channels, | |
403 int num_reverse_channels) { | |
404 if (input_sample_rate_hz == fwd_in_format_.rate() && | |
405 output_sample_rate_hz == fwd_out_format_.rate() && | |
406 reverse_sample_rate_hz == rev_in_format_.rate() && | |
407 num_input_channels == fwd_in_format_.num_channels() && | |
408 num_output_channels == fwd_out_format_.num_channels() && | |
409 num_reverse_channels == rev_in_format_.num_channels()) { | |
410 return kNoError; | 427 return kNoError; |
411 } | 428 } |
412 return InitializeLocked(input_sample_rate_hz, | 429 return InitializeLocked(processing_config); |
413 output_sample_rate_hz, | |
414 reverse_sample_rate_hz, | |
415 num_input_channels, | |
416 num_output_channels, | |
417 num_reverse_channels); | |
418 } | 430 } |
419 | 431 |
420 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 432 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
421 CriticalSectionScoped crit_scoped(crit_); | 433 CriticalSectionScoped crit_scoped(crit_); |
422 for (auto item : component_list_) { | 434 for (auto item : component_list_) { |
423 item->SetExtraOptions(config); | 435 item->SetExtraOptions(config); |
424 } | 436 } |
425 | 437 |
426 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { | 438 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { |
427 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; | 439 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; |
428 InitializeTransient(); | 440 InitializeTransient(); |
429 } | 441 } |
430 } | 442 } |
431 | 443 |
432 int AudioProcessingImpl::input_sample_rate_hz() const { | 444 int AudioProcessingImpl::input_sample_rate_hz() const { |
433 CriticalSectionScoped crit_scoped(crit_); | 445 CriticalSectionScoped crit_scoped(crit_); |
434 return fwd_in_format_.rate(); | 446 return api_format_.input_stream().sample_rate_hz(); |
435 } | 447 } |
436 | 448 |
437 int AudioProcessingImpl::sample_rate_hz() const { | 449 int AudioProcessingImpl::sample_rate_hz() const { |
438 CriticalSectionScoped crit_scoped(crit_); | 450 CriticalSectionScoped crit_scoped(crit_); |
439 return fwd_in_format_.rate(); | 451 return api_format_.input_stream().sample_rate_hz(); |
440 } | 452 } |
441 | 453 |
442 int AudioProcessingImpl::proc_sample_rate_hz() const { | 454 int AudioProcessingImpl::proc_sample_rate_hz() const { |
443 return fwd_proc_format_.rate(); | 455 return fwd_proc_format_.sample_rate_hz(); |
444 } | 456 } |
445 | 457 |
446 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 458 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
447 return split_rate_; | 459 return split_rate_; |
448 } | 460 } |
449 | 461 |
450 int AudioProcessingImpl::num_reverse_channels() const { | 462 int AudioProcessingImpl::num_reverse_channels() const { |
451 return rev_proc_format_.num_channels(); | 463 return rev_proc_format_.num_channels(); |
452 } | 464 } |
453 | 465 |
454 int AudioProcessingImpl::num_input_channels() const { | 466 int AudioProcessingImpl::num_input_channels() const { |
455 return fwd_in_format_.num_channels(); | 467 return api_format_.input_stream().num_channels(); |
456 } | 468 } |
457 | 469 |
458 int AudioProcessingImpl::num_output_channels() const { | 470 int AudioProcessingImpl::num_output_channels() const { |
459 return fwd_out_format_.num_channels(); | 471 return api_format_.output_stream().num_channels(); |
460 } | 472 } |
461 | 473 |
462 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { | 474 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { |
463 CriticalSectionScoped lock(crit_); | 475 CriticalSectionScoped lock(crit_); |
464 output_will_be_muted_ = muted; | 476 output_will_be_muted_ = muted; |
465 if (agc_manager_.get()) { | 477 if (agc_manager_.get()) { |
466 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 478 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
467 } | 479 } |
468 } | 480 } |
469 | 481 |
470 bool AudioProcessingImpl::output_will_be_muted() const { | 482 bool AudioProcessingImpl::output_will_be_muted() const { |
471 CriticalSectionScoped lock(crit_); | 483 CriticalSectionScoped lock(crit_); |
472 return output_will_be_muted_; | 484 return output_will_be_muted_; |
473 } | 485 } |
474 | 486 |
475 int AudioProcessingImpl::ProcessStream(const float* const* src, | 487 int AudioProcessingImpl::ProcessStream(const float* const* src, |
476 int samples_per_channel, | 488 int samples_per_channel, |
477 int input_sample_rate_hz, | 489 int input_sample_rate_hz, |
478 ChannelLayout input_layout, | 490 ChannelLayout input_layout, |
479 int output_sample_rate_hz, | 491 int output_sample_rate_hz, |
480 ChannelLayout output_layout, | 492 ChannelLayout output_layout, |
481 float* const* dest) { | 493 float* const* dest) { |
494 const ProcessingConfig processing_config = { | |
495 {{ | |
496 input_sample_rate_hz, ChannelsFromLayout(input_layout), | |
497 LayoutHasKeyboard(input_layout), | |
498 }, | |
499 { | |
500 output_sample_rate_hz, ChannelsFromLayout(output_layout), | |
501 LayoutHasKeyboard(output_layout), | |
502 }, | |
503 api_format_.reverse_stream()}}; | |
504 | |
505 if (samples_per_channel != | |
506 processing_config.input_stream().samples_per_channel()) { | |
507 return kBadDataLengthError; | |
508 } | |
509 return ProcessStream(src, processing_config, dest); | |
510 } | |
511 | |
512 int AudioProcessingImpl::ProcessStream( | |
513 const float* const* src, | |
514 const ProcessingConfig& processing_config, | |
515 float* const* dest) { | |
482 CriticalSectionScoped crit_scoped(crit_); | 516 CriticalSectionScoped crit_scoped(crit_); |
483 if (!src || !dest) { | 517 if (!src || !dest) { |
484 return kNullPointerError; | 518 return kNullPointerError; |
485 } | 519 } |
486 | 520 |
487 RETURN_ON_ERR(MaybeInitializeLocked(input_sample_rate_hz, | 521 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
488 output_sample_rate_hz, | 522 assert(processing_config.input_stream().samples_per_channel() == |
489 rev_in_format_.rate(), | 523 api_format_.input_stream().samples_per_channel()); |
490 ChannelsFromLayout(input_layout), | |
491 ChannelsFromLayout(output_layout), | |
492 rev_in_format_.num_channels())); | |
493 if (samples_per_channel != fwd_in_format_.samples_per_channel()) { | |
494 return kBadDataLengthError; | |
495 } | |
496 | 524 |
497 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 525 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
498 if (debug_file_->Open()) { | 526 if (debug_file_->Open()) { |
499 event_msg_->set_type(audioproc::Event::STREAM); | 527 event_msg_->set_type(audioproc::Event::STREAM); |
500 audioproc::Stream* msg = event_msg_->mutable_stream(); | 528 audioproc::Stream* msg = event_msg_->mutable_stream(); |
501 const size_t channel_size = | 529 const size_t channel_size = |
502 sizeof(float) * fwd_in_format_.samples_per_channel(); | 530 sizeof(float) * processing_config.input_stream().samples_per_channel(); |
aluebs-webrtc
2015/07/14 23:12:43
How about using api_format_.input_stream().samples
mgraczyk
2015/07/15 01:12:46
Done.
| |
503 for (int i = 0; i < fwd_in_format_.num_channels(); ++i) | 531 for (int i = 0; i < api_format_.input_stream().num_channels(); ++i) |
504 msg->add_input_channel(src[i], channel_size); | 532 msg->add_input_channel(src[i], channel_size); |
505 } | 533 } |
506 #endif | 534 #endif |
507 | 535 |
508 capture_audio_->CopyFrom(src, samples_per_channel, input_layout); | 536 capture_audio_->CopyFrom(src, api_format_.input_stream()); |
509 RETURN_ON_ERR(ProcessStreamLocked()); | 537 RETURN_ON_ERR(ProcessStreamLocked()); |
510 capture_audio_->CopyTo(fwd_out_format_.samples_per_channel(), | 538 capture_audio_->CopyTo(api_format_.output_stream(), dest); |
511 output_layout, | |
512 dest); | |
513 | 539 |
514 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 540 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
515 if (debug_file_->Open()) { | 541 if (debug_file_->Open()) { |
516 audioproc::Stream* msg = event_msg_->mutable_stream(); | 542 audioproc::Stream* msg = event_msg_->mutable_stream(); |
517 const size_t channel_size = | 543 const size_t channel_size = |
518 sizeof(float) * fwd_out_format_.samples_per_channel(); | 544 sizeof(float) * processing_config.input_stream().samples_per_channel(); |
aluebs-webrtc
2015/07/14 23:12:43
How about using api_format_.input_stream().samples
mgraczyk
2015/07/15 01:12:46
Done.
| |
519 for (int i = 0; i < fwd_out_format_.num_channels(); ++i) | 545 for (int i = 0; i < api_format_.input_stream().num_channels(); ++i) |
520 msg->add_output_channel(dest[i], channel_size); | 546 msg->add_output_channel(dest[i], channel_size); |
521 RETURN_ON_ERR(WriteMessageToDebugFile()); | 547 RETURN_ON_ERR(WriteMessageToDebugFile()); |
522 } | 548 } |
523 #endif | 549 #endif |
524 | 550 |
525 return kNoError; | 551 return kNoError; |
526 } | 552 } |
527 | 553 |
528 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { | 554 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
529 CriticalSectionScoped crit_scoped(crit_); | 555 CriticalSectionScoped crit_scoped(crit_); |
530 if (!frame) { | 556 if (!frame) { |
531 return kNullPointerError; | 557 return kNullPointerError; |
532 } | 558 } |
533 // Must be a native rate. | 559 // Must be a native rate. |
534 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 560 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
535 frame->sample_rate_hz_ != kSampleRate16kHz && | 561 frame->sample_rate_hz_ != kSampleRate16kHz && |
536 frame->sample_rate_hz_ != kSampleRate32kHz && | 562 frame->sample_rate_hz_ != kSampleRate32kHz && |
537 frame->sample_rate_hz_ != kSampleRate48kHz) { | 563 frame->sample_rate_hz_ != kSampleRate48kHz) { |
538 return kBadSampleRateError; | 564 return kBadSampleRateError; |
539 } | 565 } |
540 if (echo_control_mobile_->is_enabled() && | 566 if (echo_control_mobile_->is_enabled() && |
541 frame->sample_rate_hz_ > kSampleRate16kHz) { | 567 frame->sample_rate_hz_ > kSampleRate16kHz) { |
542 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; | 568 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; |
543 return kUnsupportedComponentError; | 569 return kUnsupportedComponentError; |
544 } | 570 } |
545 | 571 |
546 // TODO(ajm): The input and output rates and channels are currently | 572 // TODO(ajm): The input and output rates and channels are currently |
547 // constrained to be identical in the int16 interface. | 573 // constrained to be identical in the int16 interface. |
548 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_, | 574 ProcessingConfig processing_config = api_format_; |
aluebs-webrtc
2015/07/14 23:12:43
To be consistent, how about creating the Processin
mgraczyk
2015/07/15 01:12:46
I did this to be defensive against the possibility
aluebs-webrtc
2015/07/15 18:04:06
Good point! But then shouldn't you do the same in
mgraczyk
2015/07/15 20:03:19
True, Done.
| |
549 frame->sample_rate_hz_, | 575 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
550 rev_in_format_.rate(), | 576 processing_config.input_stream().set_num_channels(frame->num_channels_); |
551 frame->num_channels_, | 577 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
552 frame->num_channels_, | 578 processing_config.output_stream().set_num_channels(frame->num_channels_); |
553 rev_in_format_.num_channels())); | 579 |
554 if (frame->samples_per_channel_ != fwd_in_format_.samples_per_channel()) { | 580 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
581 if (frame->samples_per_channel_ != | |
582 api_format_.input_stream().samples_per_channel()) { | |
555 return kBadDataLengthError; | 583 return kBadDataLengthError; |
556 } | 584 } |
557 | 585 |
558 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 586 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
559 if (debug_file_->Open()) { | 587 if (debug_file_->Open()) { |
560 event_msg_->set_type(audioproc::Event::STREAM); | 588 event_msg_->set_type(audioproc::Event::STREAM); |
561 audioproc::Stream* msg = event_msg_->mutable_stream(); | 589 audioproc::Stream* msg = event_msg_->mutable_stream(); |
562 const size_t data_size = sizeof(int16_t) * | 590 const size_t data_size = |
563 frame->samples_per_channel_ * | 591 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
564 frame->num_channels_; | |
565 msg->set_input_data(frame->data_, data_size); | 592 msg->set_input_data(frame->data_, data_size); |
566 } | 593 } |
567 #endif | 594 #endif |
568 | 595 |
569 capture_audio_->DeinterleaveFrom(frame); | 596 capture_audio_->DeinterleaveFrom(frame); |
570 RETURN_ON_ERR(ProcessStreamLocked()); | 597 RETURN_ON_ERR(ProcessStreamLocked()); |
571 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); | 598 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); |
572 | 599 |
573 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 600 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
574 if (debug_file_->Open()) { | 601 if (debug_file_->Open()) { |
575 audioproc::Stream* msg = event_msg_->mutable_stream(); | 602 audioproc::Stream* msg = event_msg_->mutable_stream(); |
576 const size_t data_size = sizeof(int16_t) * | 603 const size_t data_size = |
577 frame->samples_per_channel_ * | 604 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
578 frame->num_channels_; | |
579 msg->set_output_data(frame->data_, data_size); | 605 msg->set_output_data(frame->data_, data_size); |
580 RETURN_ON_ERR(WriteMessageToDebugFile()); | 606 RETURN_ON_ERR(WriteMessageToDebugFile()); |
581 } | 607 } |
582 #endif | 608 #endif |
583 | 609 |
584 return kNoError; | 610 return kNoError; |
585 } | 611 } |
586 | 612 |
587 | |
588 int AudioProcessingImpl::ProcessStreamLocked() { | 613 int AudioProcessingImpl::ProcessStreamLocked() { |
589 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 614 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
590 if (debug_file_->Open()) { | 615 if (debug_file_->Open()) { |
591 audioproc::Stream* msg = event_msg_->mutable_stream(); | 616 audioproc::Stream* msg = event_msg_->mutable_stream(); |
592 msg->set_delay(stream_delay_ms_); | 617 msg->set_delay(stream_delay_ms_); |
593 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 618 msg->set_drift(echo_cancellation_->stream_drift_samples()); |
594 msg->set_level(gain_control()->stream_analog_level()); | 619 msg->set_level(gain_control()->stream_analog_level()); |
595 msg->set_keypress(key_pressed_); | 620 msg->set_keypress(key_pressed_); |
596 } | 621 } |
597 #endif | 622 #endif |
598 | 623 |
599 MaybeUpdateHistograms(); | 624 MaybeUpdateHistograms(); |
600 | 625 |
601 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 626 AudioBuffer* ca = capture_audio_.get(); // For brevity. |
602 if (use_new_agc_ && gain_control_->is_enabled()) { | 627 if (use_new_agc_ && gain_control_->is_enabled()) { |
603 agc_manager_->AnalyzePreProcess(ca->channels()[0], | 628 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), |
604 ca->num_channels(), | |
605 fwd_proc_format_.samples_per_channel()); | 629 fwd_proc_format_.samples_per_channel()); |
606 } | 630 } |
607 | 631 |
608 bool data_processed = is_data_processed(); | 632 bool data_processed = is_data_processed(); |
609 if (analysis_needed(data_processed)) { | 633 if (analysis_needed(data_processed)) { |
610 ca->SplitIntoFrequencyBands(); | 634 ca->SplitIntoFrequencyBands(); |
611 } | 635 } |
612 | 636 |
613 if (beamformer_enabled_) { | 637 if (beamformer_enabled_) { |
614 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 638 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); |
615 ca->set_num_channels(1); | 639 ca->set_num_channels(1); |
616 } | 640 } |
617 | 641 |
618 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 642 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); |
619 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 643 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); |
620 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 644 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); |
621 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 645 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); |
622 | 646 |
623 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { | 647 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { |
624 ca->CopyLowPassToReference(); | 648 ca->CopyLowPassToReference(); |
625 } | 649 } |
626 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); | 650 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); |
627 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); | 651 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); |
628 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); | 652 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); |
629 | 653 |
630 if (use_new_agc_ && | 654 if (use_new_agc_ && gain_control_->is_enabled() && |
631 gain_control_->is_enabled() && | |
632 (!beamformer_enabled_ || beamformer_->is_target_present())) { | 655 (!beamformer_enabled_ || beamformer_->is_target_present())) { |
633 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], | 656 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], |
634 ca->num_frames_per_band(), | 657 ca->num_frames_per_band(), split_rate_); |
635 split_rate_); | |
636 } | 658 } |
637 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); | 659 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); |
638 | 660 |
639 if (synthesis_needed(data_processed)) { | 661 if (synthesis_needed(data_processed)) { |
640 ca->MergeFrequencyBands(); | 662 ca->MergeFrequencyBands(); |
641 } | 663 } |
642 | 664 |
643 // TODO(aluebs): Investigate if the transient suppression placement should be | 665 // TODO(aluebs): Investigate if the transient suppression placement should be |
644 // before or after the AGC. | 666 // before or after the AGC. |
645 if (transient_suppressor_enabled_) { | 667 if (transient_suppressor_enabled_) { |
646 float voice_probability = | 668 float voice_probability = |
647 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; | 669 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; |
648 | 670 |
649 transient_suppressor_->Suppress(ca->channels_f()[0], | 671 transient_suppressor_->Suppress( |
650 ca->num_frames(), | 672 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), |
651 ca->num_channels(), | 673 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), |
652 ca->split_bands_const_f(0)[kBand0To8kHz], | 674 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, |
653 ca->num_frames_per_band(), | 675 key_pressed_); |
654 ca->keyboard_data(), | |
655 ca->num_keyboard_frames(), | |
656 voice_probability, | |
657 key_pressed_); | |
658 } | 676 } |
659 | 677 |
660 // The level estimator operates on the recombined data. | 678 // The level estimator operates on the recombined data. |
661 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 679 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); |
662 | 680 |
663 was_stream_delay_set_ = false; | 681 was_stream_delay_set_ = false; |
664 return kNoError; | 682 return kNoError; |
665 } | 683 } |
666 | 684 |
667 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 685 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
668 int samples_per_channel, | 686 int samples_per_channel, |
669 int sample_rate_hz, | 687 int sample_rate_hz, |
670 ChannelLayout layout) { | 688 ChannelLayout layout) { |
689 const StreamConfig reverse_config = { | |
690 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | |
691 }; | |
692 return AnalyzeReverseStream(data, reverse_config); | |
693 } | |
694 | |
695 int AudioProcessingImpl::AnalyzeReverseStream( | |
696 const float* const* data, | |
697 const StreamConfig& reverse_config) { | |
671 CriticalSectionScoped crit_scoped(crit_); | 698 CriticalSectionScoped crit_scoped(crit_); |
672 if (data == NULL) { | 699 if (data == NULL) { |
673 return kNullPointerError; | 700 return kNullPointerError; |
674 } | 701 } |
675 | 702 |
676 const int num_channels = ChannelsFromLayout(layout); | 703 if (reverse_config.num_channels() <= 0) { |
aluebs-webrtc
2015/07/14 23:12:43
Why is this needed? Is this because you don't chec
mgraczyk
2015/07/15 01:12:46
Yes.
aluebs-webrtc
2015/07/15 18:04:06
Do you think it is worth it to add these to be abl
mgraczyk
2015/07/15 20:03:19
What about clients that only use APM for it's one
aluebs-webrtc
2015/07/15 21:29:16
Ok, I was not thinking of any 1 way clients, but y
| |
677 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(), | 704 return kBadNumberChannelsError; |
678 fwd_out_format_.rate(), | 705 } |
679 sample_rate_hz, | 706 |
680 fwd_in_format_.num_channels(), | 707 const ProcessingConfig processing_config = {{api_format_.input_stream(), |
681 fwd_out_format_.num_channels(), | 708 api_format_.output_stream(), |
682 num_channels)); | 709 reverse_config}}; |
683 if (samples_per_channel != rev_in_format_.samples_per_channel()) { | 710 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
711 if (reverse_config.samples_per_channel() != | |
aluebs-webrtc
2015/07/14 23:12:43
Move this if statement to the API which receives s
mgraczyk
2015/07/15 01:12:46
Done.
| |
712 api_format_.reverse_stream().samples_per_channel()) { | |
684 return kBadDataLengthError; | 713 return kBadDataLengthError; |
685 } | 714 } |
686 | 715 |
687 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 716 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
688 if (debug_file_->Open()) { | 717 if (debug_file_->Open()) { |
689 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 718 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
690 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 719 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
691 const size_t channel_size = | 720 const size_t channel_size = |
692 sizeof(float) * rev_in_format_.samples_per_channel(); | 721 sizeof(float) * api_format_.reverse_stream().samples_per_channel(); |
693 for (int i = 0; i < num_channels; ++i) | 722 for (int i = 0; i < api_format_.reverse_stream().num_channels(); ++i) |
694 msg->add_channel(data[i], channel_size); | 723 msg->add_channel(data[i], channel_size); |
695 RETURN_ON_ERR(WriteMessageToDebugFile()); | 724 RETURN_ON_ERR(WriteMessageToDebugFile()); |
696 } | 725 } |
697 #endif | 726 #endif |
698 | 727 |
699 render_audio_->CopyFrom(data, samples_per_channel, layout); | 728 render_audio_->CopyFrom(data, api_format_.reverse_stream()); |
700 return AnalyzeReverseStreamLocked(); | 729 return AnalyzeReverseStreamLocked(); |
701 } | 730 } |
702 | 731 |
703 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 732 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
704 CriticalSectionScoped crit_scoped(crit_); | 733 CriticalSectionScoped crit_scoped(crit_); |
705 if (frame == NULL) { | 734 if (frame == NULL) { |
706 return kNullPointerError; | 735 return kNullPointerError; |
707 } | 736 } |
708 // Must be a native rate. | 737 // Must be a native rate. |
709 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 738 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
710 frame->sample_rate_hz_ != kSampleRate16kHz && | 739 frame->sample_rate_hz_ != kSampleRate16kHz && |
711 frame->sample_rate_hz_ != kSampleRate32kHz && | 740 frame->sample_rate_hz_ != kSampleRate32kHz && |
712 frame->sample_rate_hz_ != kSampleRate48kHz) { | 741 frame->sample_rate_hz_ != kSampleRate48kHz) { |
713 return kBadSampleRateError; | 742 return kBadSampleRateError; |
714 } | 743 } |
715 // This interface does not tolerate different forward and reverse rates. | 744 // This interface does not tolerate different forward and reverse rates. |
716 if (frame->sample_rate_hz_ != fwd_in_format_.rate()) { | 745 if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) { |
717 return kBadSampleRateError; | 746 return kBadSampleRateError; |
718 } | 747 } |
719 | 748 |
720 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(), | 749 if (frame->num_channels_ <= 0) { |
aluebs-webrtc
2015/07/14 23:12:43
Why is this needed? Is this because you don't chec
mgraczyk
2015/07/15 01:12:46
yes, same thing.
| |
721 fwd_out_format_.rate(), | 750 return kBadNumberChannelsError; |
722 frame->sample_rate_hz_, | 751 } |
723 fwd_in_format_.num_channels(), | 752 |
724 fwd_in_format_.num_channels(), | 753 const ProcessingConfig processing_config = {{ |
725 frame->num_channels_)); | 754 api_format_.input_stream(), |
726 if (frame->samples_per_channel_ != rev_in_format_.samples_per_channel()) { | 755 api_format_.output_stream(), |
756 { | |
757 frame->sample_rate_hz_, frame->num_channels_, | |
758 api_format_.reverse_stream().has_keyboard(), | |
759 }, | |
760 }}; | |
761 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | |
762 if (frame->samples_per_channel_ != | |
763 api_format_.reverse_stream().samples_per_channel()) { | |
727 return kBadDataLengthError; | 764 return kBadDataLengthError; |
728 } | 765 } |
729 | 766 |
730 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 767 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
731 if (debug_file_->Open()) { | 768 if (debug_file_->Open()) { |
732 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 769 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
733 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 770 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
734 const size_t data_size = sizeof(int16_t) * | 771 const size_t data_size = |
735 frame->samples_per_channel_ * | 772 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
736 frame->num_channels_; | |
737 msg->set_data(frame->data_, data_size); | 773 msg->set_data(frame->data_, data_size); |
738 RETURN_ON_ERR(WriteMessageToDebugFile()); | 774 RETURN_ON_ERR(WriteMessageToDebugFile()); |
739 } | 775 } |
740 #endif | 776 #endif |
741 | 777 |
742 render_audio_->DeinterleaveFrom(frame); | 778 render_audio_->DeinterleaveFrom(frame); |
743 return AnalyzeReverseStreamLocked(); | 779 return AnalyzeReverseStreamLocked(); |
744 } | 780 } |
745 | 781 |
746 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { | 782 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { |
747 AudioBuffer* ra = render_audio_.get(); // For brevity. | 783 AudioBuffer* ra = render_audio_.get(); // For brevity. |
748 if (rev_proc_format_.rate() == kSampleRate32kHz) { | 784 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { |
749 ra->SplitIntoFrequencyBands(); | 785 ra->SplitIntoFrequencyBands(); |
750 } | 786 } |
751 | 787 |
752 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 788 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); |
753 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 789 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); |
754 if (!use_new_agc_) { | 790 if (!use_new_agc_) { |
755 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 791 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); |
756 } | 792 } |
757 | 793 |
758 return kNoError; | 794 return kNoError; |
(...skipping 181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
940 } else if (enabled_count == 2) { | 976 } else if (enabled_count == 2) { |
941 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { | 977 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { |
942 return false; | 978 return false; |
943 } | 979 } |
944 } | 980 } |
945 return true; | 981 return true; |
946 } | 982 } |
947 | 983 |
948 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { | 984 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { |
949 // Check if we've upmixed or downmixed the audio. | 985 // Check if we've upmixed or downmixed the audio. |
950 return ((fwd_out_format_.num_channels() != fwd_in_format_.num_channels()) || | 986 return ((api_format_.output_stream().num_channels() != |
987 api_format_.input_stream().num_channels()) || | |
951 is_data_processed || transient_suppressor_enabled_); | 988 is_data_processed || transient_suppressor_enabled_); |
952 } | 989 } |
953 | 990 |
954 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { | 991 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { |
955 return (is_data_processed && (fwd_proc_format_.rate() == kSampleRate32kHz || | 992 return (is_data_processed && |
956 fwd_proc_format_.rate() == kSampleRate48kHz)); | 993 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
994 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); | |
957 } | 995 } |
958 | 996 |
959 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { | 997 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { |
960 if (!is_data_processed && !voice_detection_->is_enabled() && | 998 if (!is_data_processed && !voice_detection_->is_enabled() && |
961 !transient_suppressor_enabled_) { | 999 !transient_suppressor_enabled_) { |
962 // Only level_estimator_ is enabled. | 1000 // Only level_estimator_ is enabled. |
963 return false; | 1001 return false; |
964 } else if (fwd_proc_format_.rate() == kSampleRate32kHz || | 1002 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
965 fwd_proc_format_.rate() == kSampleRate48kHz) { | 1003 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { |
966 // Something besides level_estimator_ is enabled, and we have super-wb. | 1004 // Something besides level_estimator_ is enabled, and we have super-wb. |
967 return true; | 1005 return true; |
968 } | 1006 } |
969 return false; | 1007 return false; |
970 } | 1008 } |
971 | 1009 |
972 void AudioProcessingImpl::InitializeExperimentalAgc() { | 1010 void AudioProcessingImpl::InitializeExperimentalAgc() { |
973 if (use_new_agc_) { | 1011 if (use_new_agc_) { |
974 if (!agc_manager_.get()) { | 1012 if (!agc_manager_.get()) { |
975 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 1013 agc_manager_.reset(new AgcManagerDirect(gain_control_, |
976 gain_control_for_new_agc_.get(), | 1014 gain_control_for_new_agc_.get(), |
977 agc_startup_min_volume_)); | 1015 agc_startup_min_volume_)); |
978 } | 1016 } |
979 agc_manager_->Initialize(); | 1017 agc_manager_->Initialize(); |
980 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 1018 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
981 } | 1019 } |
982 } | 1020 } |
983 | 1021 |
984 void AudioProcessingImpl::InitializeTransient() { | 1022 void AudioProcessingImpl::InitializeTransient() { |
985 if (transient_suppressor_enabled_) { | 1023 if (transient_suppressor_enabled_) { |
986 if (!transient_suppressor_.get()) { | 1024 if (!transient_suppressor_.get()) { |
987 transient_suppressor_.reset(new TransientSuppressor()); | 1025 transient_suppressor_.reset(new TransientSuppressor()); |
988 } | 1026 } |
989 transient_suppressor_->Initialize(fwd_proc_format_.rate(), | 1027 transient_suppressor_->Initialize( |
990 split_rate_, | 1028 fwd_proc_format_.sample_rate_hz(), split_rate_, |
991 fwd_out_format_.num_channels()); | 1029 api_format_.output_stream().num_channels()); |
992 } | 1030 } |
993 } | 1031 } |
994 | 1032 |
995 void AudioProcessingImpl::InitializeBeamformer() { | 1033 void AudioProcessingImpl::InitializeBeamformer() { |
996 if (beamformer_enabled_) { | 1034 if (beamformer_enabled_) { |
997 if (!beamformer_) { | 1035 if (!beamformer_) { |
998 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); | 1036 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); |
999 } | 1037 } |
1000 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1038 beamformer_->Initialize(kChunkSizeMs, split_rate_); |
1001 } | 1039 } |
(...skipping 22 matching lines...) Expand all Loading... | |
1024 stream_delay_jumps_ = 0; // Activate counter if needed. | 1062 stream_delay_jumps_ = 0; // Activate counter if needed. |
1025 } | 1063 } |
1026 stream_delay_jumps_++; | 1064 stream_delay_jumps_++; |
1027 } | 1065 } |
1028 last_stream_delay_ms_ = stream_delay_ms_; | 1066 last_stream_delay_ms_ = stream_delay_ms_; |
1029 | 1067 |
1030 // Detect a jump in AEC system delay and log the difference. | 1068 // Detect a jump in AEC system delay and log the difference. |
1031 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); | 1069 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); |
1032 const int aec_system_delay_ms = | 1070 const int aec_system_delay_ms = |
1033 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; | 1071 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; |
1034 const int diff_aec_system_delay_ms = aec_system_delay_ms - | 1072 const int diff_aec_system_delay_ms = |
1035 last_aec_system_delay_ms_; | 1073 aec_system_delay_ms - last_aec_system_delay_ms_; |
1036 if (diff_aec_system_delay_ms > kMinDiffDelayMs && | 1074 if (diff_aec_system_delay_ms > kMinDiffDelayMs && |
1037 last_aec_system_delay_ms_ != 0) { | 1075 last_aec_system_delay_ms_ != 0) { |
1038 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", | 1076 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", |
1039 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, | 1077 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, |
1040 100); | 1078 100); |
1041 if (aec_system_delay_jumps_ == -1) { | 1079 if (aec_system_delay_jumps_ == -1) { |
1042 aec_system_delay_jumps_ = 0; // Activate counter if needed. | 1080 aec_system_delay_jumps_ = 0; // Activate counter if needed. |
1043 } | 1081 } |
1044 aec_system_delay_jumps_++; | 1082 aec_system_delay_jumps_++; |
1045 } | 1083 } |
(...skipping 19 matching lines...) Expand all Loading... | |
1065 last_aec_system_delay_ms_ = 0; | 1103 last_aec_system_delay_ms_ = 0; |
1066 } | 1104 } |
1067 | 1105 |
1068 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1106 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1069 int AudioProcessingImpl::WriteMessageToDebugFile() { | 1107 int AudioProcessingImpl::WriteMessageToDebugFile() { |
1070 int32_t size = event_msg_->ByteSize(); | 1108 int32_t size = event_msg_->ByteSize(); |
1071 if (size <= 0) { | 1109 if (size <= 0) { |
1072 return kUnspecifiedError; | 1110 return kUnspecifiedError; |
1073 } | 1111 } |
1074 #if defined(WEBRTC_ARCH_BIG_ENDIAN) | 1112 #if defined(WEBRTC_ARCH_BIG_ENDIAN) |
1075 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be | 1113 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be |
1076 // pretty safe in assuming little-endian. | 1114 // pretty safe in assuming little-endian. |
1077 #endif | 1115 #endif |
1078 | 1116 |
1079 if (!event_msg_->SerializeToString(&event_str_)) { | 1117 if (!event_msg_->SerializeToString(&event_str_)) { |
1080 return kUnspecifiedError; | 1118 return kUnspecifiedError; |
1081 } | 1119 } |
1082 | 1120 |
1083 // Write message preceded by its size. | 1121 // Write message preceded by its size. |
1084 if (!debug_file_->Write(&size, sizeof(int32_t))) { | 1122 if (!debug_file_->Write(&size, sizeof(int32_t))) { |
1085 return kFileError; | 1123 return kFileError; |
1086 } | 1124 } |
1087 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { | 1125 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { |
1088 return kFileError; | 1126 return kFileError; |
1089 } | 1127 } |
1090 | 1128 |
1091 event_msg_->Clear(); | 1129 event_msg_->Clear(); |
1092 | 1130 |
1093 return kNoError; | 1131 return kNoError; |
1094 } | 1132 } |
1095 | 1133 |
1096 int AudioProcessingImpl::WriteInitMessage() { | 1134 int AudioProcessingImpl::WriteInitMessage() { |
1097 event_msg_->set_type(audioproc::Event::INIT); | 1135 event_msg_->set_type(audioproc::Event::INIT); |
1098 audioproc::Init* msg = event_msg_->mutable_init(); | 1136 audioproc::Init* msg = event_msg_->mutable_init(); |
1099 msg->set_sample_rate(fwd_in_format_.rate()); | 1137 msg->set_sample_rate(api_format_.input_stream().sample_rate_hz()); |
1100 msg->set_num_input_channels(fwd_in_format_.num_channels()); | 1138 msg->set_num_input_channels(api_format_.input_stream().num_channels()); |
1101 msg->set_num_output_channels(fwd_out_format_.num_channels()); | 1139 msg->set_num_output_channels(api_format_.output_stream().num_channels()); |
1102 msg->set_num_reverse_channels(rev_in_format_.num_channels()); | 1140 msg->set_num_reverse_channels(api_format_.reverse_stream().num_channels()); |
1103 msg->set_reverse_sample_rate(rev_in_format_.rate()); | 1141 msg->set_reverse_sample_rate(api_format_.reverse_stream().sample_rate_hz()); |
1104 msg->set_output_sample_rate(fwd_out_format_.rate()); | 1142 msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz()); |
1105 | 1143 |
1106 int err = WriteMessageToDebugFile(); | 1144 int err = WriteMessageToDebugFile(); |
1107 if (err != kNoError) { | 1145 if (err != kNoError) { |
1108 return err; | 1146 return err; |
1109 } | 1147 } |
1110 | 1148 |
1111 return kNoError; | 1149 return kNoError; |
1112 } | 1150 } |
1113 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1151 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1114 | 1152 |
1115 } // namespace webrtc | 1153 } // namespace webrtc |
OLD | NEW |