OLD | NEW |
1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media/media_stream_audio_processor.h" | 5 #include "content/renderer/media/media_stream_audio_processor.h" |
6 | 6 |
7 #include "base/command_line.h" | 7 #include "base/command_line.h" |
8 #include "base/debug/trace_event.h" | 8 #include "base/debug/trace_event.h" |
9 #include "base/metrics/histogram.h" | 9 #include "base/metrics/histogram.h" |
10 #include "content/public/common/content_switches.h" | 10 #include "content/public/common/content_switches.h" |
(...skipping 13 matching lines...) Expand all Loading... |
24 namespace { | 24 namespace { |
25 | 25 |
26 using webrtc::AudioProcessing; | 26 using webrtc::AudioProcessing; |
27 | 27 |
28 #if defined(OS_ANDROID) | 28 #if defined(OS_ANDROID) |
29 const int kAudioProcessingSampleRate = 16000; | 29 const int kAudioProcessingSampleRate = 16000; |
30 #else | 30 #else |
31 const int kAudioProcessingSampleRate = 32000; | 31 const int kAudioProcessingSampleRate = 32000; |
32 #endif | 32 #endif |
33 const int kAudioProcessingNumberOfChannels = 1; | 33 const int kAudioProcessingNumberOfChannels = 1; |
34 const AudioProcessing::ChannelLayout kAudioProcessingChannelLayout = | |
35 AudioProcessing::kMono; | |
36 | 34 |
37 const int kMaxNumberOfBuffersInFifo = 2; | 35 AudioProcessing::ChannelLayout MapLayout(media::ChannelLayout media_layout) { |
| 36 switch (media_layout) { |
| 37 case media::CHANNEL_LAYOUT_MONO: |
| 38 return AudioProcessing::kMono; |
| 39 case media::CHANNEL_LAYOUT_STEREO: |
| 40 return AudioProcessing::kStereo; |
| 41 case media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC: |
| 42 return AudioProcessing::kStereoAndKeyboard; |
| 43 default: |
| 44 NOTREACHED() << "Layout not supported: " << media_layout; |
| 45 return AudioProcessing::kMono; |
| 46 } |
| 47 } |
| 48 |
| 49 AudioProcessing::ChannelLayout ChannelsToLayout(int num_channels) { |
| 50 switch (num_channels) { |
| 51 case 1: |
| 52 return AudioProcessing::kMono; |
| 53 case 2: |
| 54 return AudioProcessing::kStereo; |
| 55 default: |
| 56 NOTREACHED() << "Channels not supported: " << num_channels; |
| 57 return AudioProcessing::kMono; |
| 58 } |
| 59 } |
38 | 60 |
39 // Used by UMA histograms and entries shouldn't be re-ordered or removed. | 61 // Used by UMA histograms and entries shouldn't be re-ordered or removed. |
40 enum AudioTrackProcessingStates { | 62 enum AudioTrackProcessingStates { |
41 AUDIO_PROCESSING_ENABLED = 0, | 63 AUDIO_PROCESSING_ENABLED = 0, |
42 AUDIO_PROCESSING_DISABLED, | 64 AUDIO_PROCESSING_DISABLED, |
43 AUDIO_PROCESSING_IN_WEBRTC, | 65 AUDIO_PROCESSING_IN_WEBRTC, |
44 AUDIO_PROCESSING_MAX | 66 AUDIO_PROCESSING_MAX |
45 }; | 67 }; |
46 | 68 |
47 void RecordProcessingState(AudioTrackProcessingStates state) { | 69 void RecordProcessingState(AudioTrackProcessingStates state) { |
48 UMA_HISTOGRAM_ENUMERATION("Media.AudioTrackProcessingStates", | 70 UMA_HISTOGRAM_ENUMERATION("Media.AudioTrackProcessingStates", |
49 state, AUDIO_PROCESSING_MAX); | 71 state, AUDIO_PROCESSING_MAX); |
50 } | 72 } |
51 | 73 |
52 } // namespace | 74 } // namespace |
53 | 75 |
54 class MediaStreamAudioProcessor::MediaStreamAudioConverter | 76 // Wraps AudioBus to provide access to the array of channel pointers, since this |
55 : public media::AudioConverter::InputCallback { | 77 // is the type webrtc::AudioProcessing deals in. The array is refreshed on every |
| 78 // channel_ptrs() call, and will be valid until the underlying AudioBus pointers |
| 79 // are changed, e.g. through calls to SetChannelData() or SwapChannels(). |
| 80 // |
| 81 // All methods are called on one of the capture or render audio threads |
| 82 // exclusively. |
| 83 class MediaStreamAudioBus { |
56 public: | 84 public: |
57 MediaStreamAudioConverter(const media::AudioParameters& source_params, | 85 MediaStreamAudioBus(int channels, int frames) |
58 const media::AudioParameters& sink_params) | 86 : bus_(media::AudioBus::Create(channels, frames)), |
59 : source_params_(source_params), | 87 channel_ptrs_(new float*[channels]) { |
60 sink_params_(sink_params), | 88 // May be created in the main render thread and used in the audio threads. |
61 audio_converter_(source_params, sink_params_, false) { | |
62 // An instance of MediaStreamAudioConverter may be created in the main | |
63 // render thread and used in the audio thread, for example, the | |
64 // |MediaStreamAudioProcessor::capture_converter_|. | |
65 thread_checker_.DetachFromThread(); | 89 thread_checker_.DetachFromThread(); |
66 audio_converter_.AddInput(this); | |
67 | |
68 // Create and initialize audio fifo and audio bus wrapper. | |
69 // The size of the FIFO should be at least twice of the source buffer size | |
70 // or twice of the sink buffer size. Also, FIFO needs to have enough space | |
71 // to store pre-processed data before passing the data to | |
72 // webrtc::AudioProcessing, which requires 10ms as packet size. | |
73 int max_frame_size = std::max(source_params_.frames_per_buffer(), | |
74 sink_params_.frames_per_buffer()); | |
75 int buffer_size = std::max( | |
76 kMaxNumberOfBuffersInFifo * max_frame_size, | |
77 kMaxNumberOfBuffersInFifo * source_params_.sample_rate() / 100); | |
78 fifo_.reset(new media::AudioFifo(source_params_.channels(), buffer_size)); | |
79 | |
80 // TODO(xians): Use CreateWrapper to save one memcpy. | |
81 audio_wrapper_ = media::AudioBus::Create(sink_params_.channels(), | |
82 sink_params_.frames_per_buffer()); | |
83 } | 90 } |
84 | 91 |
85 virtual ~MediaStreamAudioConverter() { | 92 media::AudioBus* bus() { |
86 audio_converter_.RemoveInput(this); | 93 DCHECK(thread_checker_.CalledOnValidThread()); |
| 94 return bus_.get(); |
87 } | 95 } |
88 | 96 |
89 void Push(const media::AudioBus* audio_source) { | 97 float* const* channel_ptrs() { |
90 // Called on the audio thread, which is the capture audio thread for | |
91 // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread | |
92 // for |MediaStreamAudioProcessor::render_converter_|. | |
93 // And it must be the same thread as calling Convert(). | |
94 DCHECK(thread_checker_.CalledOnValidThread()); | 98 DCHECK(thread_checker_.CalledOnValidThread()); |
95 fifo_->Push(audio_source); | 99 for (int i = 0; i < bus_->channels(); ++i) { |
| 100 channel_ptrs_[i] = bus_->channel(i); |
| 101 } |
| 102 return channel_ptrs_.get(); |
96 } | 103 } |
97 | 104 |
98 bool Convert(webrtc::AudioFrame* out, bool audio_mirroring) { | 105 private: |
99 // Called on the audio thread, which is the capture audio thread for | 106 base::ThreadChecker thread_checker_; |
100 // |MediaStreamAudioProcessor::capture_converter_|, and render audio thread | 107 scoped_ptr<media::AudioBus> bus_; |
101 // for |MediaStreamAudioProcessor::render_converter_|. | 108 scoped_ptr<float*[]> channel_ptrs_; |
102 DCHECK(thread_checker_.CalledOnValidThread()); | 109 }; |
103 // Return false if there is not enough data in the FIFO, this happens when | 110 |
104 // fifo_->frames() / source_params_.sample_rate() is less than | 111 // Wraps AudioFifo to provide a cleaner interface to MediaStreamAudioProcessor. |
105 // sink_params.frames_per_buffer() / sink_params.sample_rate(). | 112 // It avoids the FIFO when the source and destination frames match. All methods |
106 if (fifo_->frames() * sink_params_.sample_rate() < | 113 // are called on one of the capture or render audio threads exclusively. |
107 sink_params_.frames_per_buffer() * source_params_.sample_rate()) { | 114 class MediaStreamAudioFifo { |
108 return false; | 115 public: |
| 116 MediaStreamAudioFifo(int channels, int source_frames, |
| 117 int destination_frames) |
| 118 : source_frames_(source_frames), |
| 119 destination_(new MediaStreamAudioBus(channels, destination_frames)), |
| 120 data_available_(false) { |
| 121 if (source_frames != destination_frames) { |
| 122 // Since we require every Push to be followed by as many Consumes as |
| 123 // possible, twice the larger of the two is a (probably) loose upper bound |
| 124 // on the FIFO size. |
| 125 const int fifo_frames = 2 * std::max(source_frames, destination_frames); |
| 126 fifo_.reset(new media::AudioFifo(channels, fifo_frames)); |
109 } | 127 } |
110 | 128 |
111 // Convert data to the output format, this will trigger ProvideInput(). | 129 // May be created in the main render thread and used in the audio threads. |
112 audio_converter_.Convert(audio_wrapper_.get()); | 130 thread_checker_.DetachFromThread(); |
113 DCHECK_EQ(audio_wrapper_->frames(), sink_params_.frames_per_buffer()); | 131 } |
114 | 132 |
115 // Swap channels before interleaving the data if |audio_mirroring| is | 133 void Push(const media::AudioBus* source) { |
116 // set to true. | 134 DCHECK(thread_checker_.CalledOnValidThread()); |
117 if (audio_mirroring && | 135 DCHECK_EQ(source->channels(), destination_->bus()->channels()); |
118 sink_params_.channel_layout() == media::CHANNEL_LAYOUT_STEREO) { | 136 DCHECK_EQ(source->frames(), source_frames_); |
119 // Swap the first and second channels. | 137 |
120 audio_wrapper_->SwapChannels(0, 1); | 138 if (fifo_) { |
| 139 fifo_->Push(source); |
| 140 } else { |
| 141 source->CopyTo(destination_->bus()); |
| 142 data_available_ = true; |
| 143 } |
| 144 } |
| 145 |
| 146 // Returns true if there are destination_frames() of data available to be |
| 147 // consumed, and otherwise false. |
| 148 bool Consume(MediaStreamAudioBus** destination) { |
| 149 DCHECK(thread_checker_.CalledOnValidThread()); |
| 150 |
| 151 if (fifo_) { |
| 152 if (fifo_->frames() < destination_->bus()->frames()) |
| 153 return false; |
| 154 |
| 155 fifo_->Consume(destination_->bus(), 0, destination_->bus()->frames()); |
| 156 } else { |
| 157 if (!data_available_) |
| 158 return false; |
| 159 |
| 160 // The data was already copied to |destination_| in this case. |
| 161 data_available_ = false; |
121 } | 162 } |
122 | 163 |
123 // TODO(xians): Figure out a better way to handle the interleaved and | 164 *destination = destination_.get(); |
124 // deinterleaved format switching. | |
125 audio_wrapper_->ToInterleaved(audio_wrapper_->frames(), | |
126 sink_params_.bits_per_sample() / 8, | |
127 out->data_); | |
128 | |
129 out->samples_per_channel_ = sink_params_.frames_per_buffer(); | |
130 out->sample_rate_hz_ = sink_params_.sample_rate(); | |
131 out->speech_type_ = webrtc::AudioFrame::kNormalSpeech; | |
132 out->vad_activity_ = webrtc::AudioFrame::kVadUnknown; | |
133 out->num_channels_ = sink_params_.channels(); | |
134 | |
135 return true; | 165 return true; |
136 } | 166 } |
137 | 167 |
138 const media::AudioParameters& source_parameters() const { | |
139 return source_params_; | |
140 } | |
141 const media::AudioParameters& sink_parameters() const { | |
142 return sink_params_; | |
143 } | |
144 | |
145 private: | 168 private: |
146 // AudioConverter::InputCallback implementation. | |
147 virtual double ProvideInput(media::AudioBus* audio_bus, | |
148 base::TimeDelta buffer_delay) OVERRIDE { | |
149 // Called on realtime audio thread. | |
150 // TODO(xians): Figure out why the first Convert() triggers ProvideInput | |
151 // two times. | |
152 if (fifo_->frames() < audio_bus->frames()) | |
153 return 0; | |
154 | |
155 fifo_->Consume(audio_bus, 0, audio_bus->frames()); | |
156 | |
157 // Return 1.0 to indicate no volume scaling on the data. | |
158 return 1.0; | |
159 } | |
160 | |
161 base::ThreadChecker thread_checker_; | 169 base::ThreadChecker thread_checker_; |
162 const media::AudioParameters source_params_; | 170 const int source_frames_; // For a DCHECK. |
163 const media::AudioParameters sink_params_; | 171 scoped_ptr<MediaStreamAudioBus> destination_; |
164 | |
165 // TODO(xians): consider using SincResampler to save some memcpy. | |
166 // Handles mixing and resampling between input and output parameters. | |
167 media::AudioConverter audio_converter_; | |
168 scoped_ptr<media::AudioBus> audio_wrapper_; | |
169 scoped_ptr<media::AudioFifo> fifo_; | 172 scoped_ptr<media::AudioFifo> fifo_; |
| 173 // Only used when the FIFO is disabled; |
| 174 bool data_available_; |
170 }; | 175 }; |
171 | 176 |
172 bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { | 177 bool MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() { |
173 return !CommandLine::ForCurrentProcess()->HasSwitch( | 178 return !CommandLine::ForCurrentProcess()->HasSwitch( |
174 switches::kDisableAudioTrackProcessing); | 179 switches::kDisableAudioTrackProcessing); |
175 } | 180 } |
176 | 181 |
177 MediaStreamAudioProcessor::MediaStreamAudioProcessor( | 182 MediaStreamAudioProcessor::MediaStreamAudioProcessor( |
178 const blink::WebMediaConstraints& constraints, | 183 const blink::WebMediaConstraints& constraints, |
179 int effects, | 184 int effects, |
(...skipping 15 matching lines...) Expand all Loading... |
195 aec_dump_message_filter_->AddDelegate(this); | 200 aec_dump_message_filter_->AddDelegate(this); |
196 } | 201 } |
197 } | 202 } |
198 | 203 |
199 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { | 204 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { |
200 DCHECK(main_thread_checker_.CalledOnValidThread()); | 205 DCHECK(main_thread_checker_.CalledOnValidThread()); |
201 Stop(); | 206 Stop(); |
202 } | 207 } |
203 | 208 |
204 void MediaStreamAudioProcessor::OnCaptureFormatChanged( | 209 void MediaStreamAudioProcessor::OnCaptureFormatChanged( |
205 const media::AudioParameters& source_params) { | 210 const media::AudioParameters& input_format) { |
206 DCHECK(main_thread_checker_.CalledOnValidThread()); | 211 DCHECK(main_thread_checker_.CalledOnValidThread()); |
207 // There is no need to hold a lock here since the caller guarantees that | 212 // There is no need to hold a lock here since the caller guarantees that |
208 // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks | 213 // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks |
209 // on the capture thread. | 214 // on the capture thread. |
210 InitializeCaptureConverter(source_params); | 215 InitializeCaptureFifo(input_format); |
211 | 216 |
212 // Reset the |capture_thread_checker_| since the capture data will come from | 217 // Reset the |capture_thread_checker_| since the capture data will come from |
213 // a new capture thread. | 218 // a new capture thread. |
214 capture_thread_checker_.DetachFromThread(); | 219 capture_thread_checker_.DetachFromThread(); |
215 } | 220 } |
216 | 221 |
217 void MediaStreamAudioProcessor::PushCaptureData( | 222 void MediaStreamAudioProcessor::PushCaptureData( |
218 const media::AudioBus* audio_source) { | 223 const media::AudioBus* audio_source) { |
219 DCHECK(capture_thread_checker_.CalledOnValidThread()); | 224 DCHECK(capture_thread_checker_.CalledOnValidThread()); |
220 DCHECK_EQ(audio_source->channels(), | |
221 capture_converter_->source_parameters().channels()); | |
222 DCHECK_EQ(audio_source->frames(), | |
223 capture_converter_->source_parameters().frames_per_buffer()); | |
224 | 225 |
225 capture_converter_->Push(audio_source); | 226 capture_fifo_->Push(audio_source); |
226 } | 227 } |
227 | 228 |
228 bool MediaStreamAudioProcessor::ProcessAndConsumeData( | 229 bool MediaStreamAudioProcessor::ProcessAndConsumeData( |
229 base::TimeDelta capture_delay, int volume, bool key_pressed, | 230 base::TimeDelta capture_delay, int volume, bool key_pressed, |
230 int* new_volume, int16** out) { | 231 int* new_volume, int16** out) { |
231 DCHECK(capture_thread_checker_.CalledOnValidThread()); | 232 DCHECK(capture_thread_checker_.CalledOnValidThread()); |
232 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData"); | 233 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData"); |
233 | 234 |
234 if (!capture_converter_->Convert(&capture_frame_, audio_mirroring_)) | 235 MediaStreamAudioBus* process_bus; |
| 236 if (!capture_fifo_->Consume(&process_bus)) |
235 return false; | 237 return false; |
236 | 238 |
237 *new_volume = ProcessData(&capture_frame_, capture_delay, volume, | 239 // Use the process bus directly if audio processing is disabled. |
238 key_pressed); | 240 MediaStreamAudioBus* output_bus = process_bus; |
239 *out = capture_frame_.data_; | 241 *new_volume = 0; |
| 242 if (audio_processing_) { |
| 243 output_bus = output_bus_.get(); |
| 244 *new_volume = ProcessData(process_bus->channel_ptrs(), |
| 245 process_bus->bus()->frames(), capture_delay, |
| 246 volume, key_pressed, output_bus->channel_ptrs()); |
| 247 } |
| 248 |
| 249 // Swap channels before interleaving the data. |
| 250 if (audio_mirroring_ && |
| 251 output_format_.channel_layout() == media::CHANNEL_LAYOUT_STEREO) { |
| 252 // Swap the first and second channels. |
| 253 output_bus->bus()->SwapChannels(0, 1); |
| 254 } |
| 255 |
| 256 output_bus->bus()->ToInterleaved(output_bus->bus()->frames(), |
| 257 sizeof(int16), |
| 258 output_data_.get()); |
| 259 *out = output_data_.get(); |
240 | 260 |
241 return true; | 261 return true; |
242 } | 262 } |
243 | 263 |
244 void MediaStreamAudioProcessor::Stop() { | 264 void MediaStreamAudioProcessor::Stop() { |
245 DCHECK(main_thread_checker_.CalledOnValidThread()); | 265 DCHECK(main_thread_checker_.CalledOnValidThread()); |
246 if (stopped_) | 266 if (stopped_) |
247 return; | 267 return; |
248 | 268 |
249 stopped_ = true; | 269 stopped_ = true; |
250 | 270 |
251 if (aec_dump_message_filter_) { | 271 if (aec_dump_message_filter_) { |
252 aec_dump_message_filter_->RemoveDelegate(this); | 272 aec_dump_message_filter_->RemoveDelegate(this); |
253 aec_dump_message_filter_ = NULL; | 273 aec_dump_message_filter_ = NULL; |
254 } | 274 } |
255 | 275 |
256 if (!audio_processing_.get()) | 276 if (!audio_processing_.get()) |
257 return; | 277 return; |
258 | 278 |
259 StopEchoCancellationDump(audio_processing_.get()); | 279 StopEchoCancellationDump(audio_processing_.get()); |
260 | 280 |
261 if (playout_data_source_) { | 281 if (playout_data_source_) { |
262 playout_data_source_->RemovePlayoutSink(this); | 282 playout_data_source_->RemovePlayoutSink(this); |
263 playout_data_source_ = NULL; | 283 playout_data_source_ = NULL; |
264 } | 284 } |
265 } | 285 } |
266 | 286 |
267 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { | 287 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { |
268 return capture_converter_->source_parameters(); | 288 return input_format_; |
269 } | 289 } |
270 | 290 |
271 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { | 291 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { |
272 return capture_converter_->sink_parameters(); | 292 return output_format_; |
273 } | 293 } |
274 | 294 |
275 void MediaStreamAudioProcessor::OnAecDumpFile( | 295 void MediaStreamAudioProcessor::OnAecDumpFile( |
276 const IPC::PlatformFileForTransit& file_handle) { | 296 const IPC::PlatformFileForTransit& file_handle) { |
277 DCHECK(main_thread_checker_.CalledOnValidThread()); | 297 DCHECK(main_thread_checker_.CalledOnValidThread()); |
278 | 298 |
279 base::File file = IPC::PlatformFileForTransitToFile(file_handle); | 299 base::File file = IPC::PlatformFileForTransitToFile(file_handle); |
280 DCHECK(file.IsValid()); | 300 DCHECK(file.IsValid()); |
281 | 301 |
282 if (audio_processing_) | 302 if (audio_processing_) |
(...skipping 18 matching lines...) Expand all Loading... |
301 int audio_delay_milliseconds) { | 321 int audio_delay_milliseconds) { |
302 DCHECK(render_thread_checker_.CalledOnValidThread()); | 322 DCHECK(render_thread_checker_.CalledOnValidThread()); |
303 DCHECK(audio_processing_->echo_control_mobile()->is_enabled() ^ | 323 DCHECK(audio_processing_->echo_control_mobile()->is_enabled() ^ |
304 audio_processing_->echo_cancellation()->is_enabled()); | 324 audio_processing_->echo_cancellation()->is_enabled()); |
305 | 325 |
306 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::OnPlayoutData"); | 326 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::OnPlayoutData"); |
307 DCHECK_LT(audio_delay_milliseconds, | 327 DCHECK_LT(audio_delay_milliseconds, |
308 std::numeric_limits<base::subtle::Atomic32>::max()); | 328 std::numeric_limits<base::subtle::Atomic32>::max()); |
309 base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds); | 329 base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds); |
310 | 330 |
311 InitializeRenderConverterIfNeeded(sample_rate, audio_bus->channels(), | 331 InitializeRenderFifoIfNeeded(sample_rate, audio_bus->channels(), |
312 audio_bus->frames()); | 332 audio_bus->frames()); |
313 | 333 |
314 render_converter_->Push(audio_bus); | 334 render_fifo_->Push(audio_bus); |
315 while (render_converter_->Convert(&render_frame_, false)) | 335 MediaStreamAudioBus* analysis_bus; |
316 audio_processing_->AnalyzeReverseStream(&render_frame_); | 336 while (render_fifo_->Consume(&analysis_bus)) { |
| 337 audio_processing_->AnalyzeReverseStream( |
| 338 analysis_bus->channel_ptrs(), |
| 339 analysis_bus->bus()->frames(), |
| 340 sample_rate, |
| 341 ChannelsToLayout(audio_bus->channels())); |
| 342 } |
317 } | 343 } |
318 | 344 |
319 void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() { | 345 void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() { |
320 DCHECK(main_thread_checker_.CalledOnValidThread()); | 346 DCHECK(main_thread_checker_.CalledOnValidThread()); |
321 // There is no need to hold a lock here since the caller guarantees that | 347 // There is no need to hold a lock here since the caller guarantees that |
322 // there is no more OnPlayoutData() callback on the render thread. | 348 // there is no more OnPlayoutData() callback on the render thread. |
323 render_thread_checker_.DetachFromThread(); | 349 render_thread_checker_.DetachFromThread(); |
324 render_converter_.reset(); | 350 render_fifo_.reset(); |
325 } | 351 } |
326 | 352 |
327 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { | 353 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { |
328 stats->typing_noise_detected = | 354 stats->typing_noise_detected = |
329 (base::subtle::Acquire_Load(&typing_detected_) != false); | 355 (base::subtle::Acquire_Load(&typing_detected_) != false); |
330 GetAecStats(audio_processing_.get(), stats); | 356 GetAecStats(audio_processing_.get(), stats); |
331 } | 357 } |
332 | 358 |
333 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( | 359 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( |
334 const blink::WebMediaConstraints& constraints, int effects) { | 360 const blink::WebMediaConstraints& constraints, int effects) { |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
377 // Return immediately if no goog constraint is enabled. | 403 // Return immediately if no goog constraint is enabled. |
378 if (!echo_cancellation && !goog_experimental_aec && !goog_ns && | 404 if (!echo_cancellation && !goog_experimental_aec && !goog_ns && |
379 !goog_high_pass_filter && !goog_typing_detection && | 405 !goog_high_pass_filter && !goog_typing_detection && |
380 !goog_agc && !goog_experimental_ns) { | 406 !goog_agc && !goog_experimental_ns) { |
381 RecordProcessingState(AUDIO_PROCESSING_DISABLED); | 407 RecordProcessingState(AUDIO_PROCESSING_DISABLED); |
382 return; | 408 return; |
383 } | 409 } |
384 | 410 |
385 // Create and configure the webrtc::AudioProcessing. | 411 // Create and configure the webrtc::AudioProcessing. |
386 audio_processing_.reset(webrtc::AudioProcessing::Create()); | 412 audio_processing_.reset(webrtc::AudioProcessing::Create()); |
387 CHECK_EQ(0, audio_processing_->Initialize(kAudioProcessingSampleRate, | |
388 kAudioProcessingSampleRate, | |
389 kAudioProcessingSampleRate, | |
390 kAudioProcessingChannelLayout, | |
391 kAudioProcessingChannelLayout, | |
392 kAudioProcessingChannelLayout)); | |
393 | 413 |
394 // Enable the audio processing components. | 414 // Enable the audio processing components. |
395 if (echo_cancellation) { | 415 if (echo_cancellation) { |
396 EnableEchoCancellation(audio_processing_.get()); | 416 EnableEchoCancellation(audio_processing_.get()); |
397 | 417 |
398 if (goog_experimental_aec) | 418 if (goog_experimental_aec) |
399 EnableExperimentalEchoCancellation(audio_processing_.get()); | 419 EnableExperimentalEchoCancellation(audio_processing_.get()); |
400 | 420 |
401 if (playout_data_source_) | 421 if (playout_data_source_) |
402 playout_data_source_->AddPlayoutSink(this); | 422 playout_data_source_->AddPlayoutSink(this); |
(...skipping 14 matching lines...) Expand all Loading... |
417 typing_detector_.reset(new webrtc::TypingDetection()); | 437 typing_detector_.reset(new webrtc::TypingDetection()); |
418 EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); | 438 EnableTypingDetection(audio_processing_.get(), typing_detector_.get()); |
419 } | 439 } |
420 | 440 |
421 if (goog_agc) | 441 if (goog_agc) |
422 EnableAutomaticGainControl(audio_processing_.get()); | 442 EnableAutomaticGainControl(audio_processing_.get()); |
423 | 443 |
424 RecordProcessingState(AUDIO_PROCESSING_ENABLED); | 444 RecordProcessingState(AUDIO_PROCESSING_ENABLED); |
425 } | 445 } |
426 | 446 |
427 void MediaStreamAudioProcessor::InitializeCaptureConverter( | 447 void MediaStreamAudioProcessor::InitializeCaptureFifo( |
428 const media::AudioParameters& source_params) { | 448 const media::AudioParameters& input_format) { |
429 DCHECK(main_thread_checker_.CalledOnValidThread()); | 449 DCHECK(main_thread_checker_.CalledOnValidThread()); |
430 DCHECK(source_params.IsValid()); | 450 DCHECK(input_format.IsValid()); |
| 451 input_format_ = input_format; |
431 | 452 |
432 // Create and initialize audio converter for the source data. | 453 // TODO(ajm): For now, we assume fixed parameters for the output when audio |
433 // When the webrtc AudioProcessing is enabled, the sink format of the | 454 // processing is enabled, to match the previous behavior. We should either |
434 // converter will be the same as the post-processed data format, which is | 455 // use the input parameters (in which case, audio processing will convert |
435 // 32k mono for desktops and 16k mono for Android. When the AudioProcessing | 456 // at output) or ideally, have a backchannel from the sink to know what |
436 // is disabled, the sink format will be the same as the source format. | 457 // format it would prefer. |
437 const int sink_sample_rate = audio_processing_ ? | 458 const int output_sample_rate = audio_processing_ ? |
438 kAudioProcessingSampleRate : source_params.sample_rate(); | 459 kAudioProcessingSampleRate : input_format.sample_rate(); |
439 const media::ChannelLayout sink_channel_layout = audio_processing_ ? | 460 const media::ChannelLayout output_channel_layout = audio_processing_ ? |
440 media::GuessChannelLayout(kAudioProcessingNumberOfChannels) : | 461 media::GuessChannelLayout(kAudioProcessingNumberOfChannels) : |
441 source_params.channel_layout(); | 462 input_format.channel_layout(); |
442 | 463 |
443 // WebRtc AudioProcessing requires 10ms as its packet size. We use this | 464 // webrtc::AudioProcessing requires a 10 ms chunk size. We use this native |
444 // native size when processing is enabled. While processing is disabled, and | 465 // size when processing is enabled. When disabled we use the same size as |
445 // the source is running with a buffer size smaller than 10ms buffer, we use | 466 // the source if less than 10 ms. |
446 // same buffer size as the incoming format to avoid extra FIFO for WebAudio. | 467 // |
447 int sink_buffer_size = sink_sample_rate / 100; | 468 // TODO(ajm): This conditional buffer size appears to be assuming knowledge of |
448 if (!audio_processing_ && | 469 // the sink based on the source parameters. PeerConnection sinks seem to want |
449 source_params.frames_per_buffer() < sink_buffer_size) { | 470 // 10 ms chunks regardless, while WebAudio sinks want less, and we're assuming |
450 sink_buffer_size = source_params.frames_per_buffer(); | 471 // we can identify WebAudio sinks by the input chunk size. Less fragile would |
| 472 // be to have the sink actually tell us how much it wants (as in the above |
| 473 // TODO). |
| 474 int processing_frames = input_format.sample_rate() / 100; |
| 475 int output_frames = output_sample_rate / 100; |
| 476 if (!audio_processing_ && input_format.frames_per_buffer() < output_frames) { |
| 477 processing_frames = input_format.frames_per_buffer(); |
| 478 output_frames = processing_frames; |
451 } | 479 } |
452 | 480 |
453 media::AudioParameters sink_params( | 481 output_format_ = media::AudioParameters( |
454 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, sink_channel_layout, | 482 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, |
455 sink_sample_rate, 16, sink_buffer_size); | 483 output_channel_layout, |
456 capture_converter_.reset( | 484 output_sample_rate, |
457 new MediaStreamAudioConverter(source_params, sink_params)); | 485 16, |
| 486 output_frames); |
| 487 |
| 488 capture_fifo_.reset( |
| 489 new MediaStreamAudioFifo(input_format.channels(), |
| 490 input_format.frames_per_buffer(), |
| 491 processing_frames)); |
| 492 |
| 493 if (audio_processing_) { |
| 494 output_bus_.reset(new MediaStreamAudioBus(output_format_.channels(), |
| 495 output_frames)); |
| 496 } |
| 497 output_data_.reset(new int16[output_format_.GetBytesPerBuffer() / |
| 498 sizeof(int16)]); |
458 } | 499 } |
459 | 500 |
460 void MediaStreamAudioProcessor::InitializeRenderConverterIfNeeded( | 501 void MediaStreamAudioProcessor::InitializeRenderFifoIfNeeded( |
461 int sample_rate, int number_of_channels, int frames_per_buffer) { | 502 int sample_rate, int number_of_channels, int frames_per_buffer) { |
462 DCHECK(render_thread_checker_.CalledOnValidThread()); | 503 DCHECK(render_thread_checker_.CalledOnValidThread()); |
463 // TODO(xians): Figure out if we need to handle the buffer size change. | 504 if (render_fifo_.get() && |
464 if (render_converter_.get() && | 505 render_format_.sample_rate() == sample_rate && |
465 render_converter_->source_parameters().sample_rate() == sample_rate && | 506 render_format_.channels() == number_of_channels && |
466 render_converter_->source_parameters().channels() == number_of_channels) { | 507 render_format_.frames_per_buffer() == frames_per_buffer) { |
467 // Do nothing if the |render_converter_| has been setup properly. | 508 // Do nothing if the |render_fifo_| has been setup properly. |
468 return; | 509 return; |
469 } | 510 } |
470 | 511 |
471 // Create and initialize audio converter for the render data. | 512 render_format_ = media::AudioParameters( |
472 // webrtc::AudioProcessing accepts the same format as what it uses to process | |
473 // capture data, which is 32k mono for desktops and 16k mono for Android. | |
474 media::AudioParameters source_params( | |
475 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, | 513 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, |
476 media::GuessChannelLayout(number_of_channels), sample_rate, 16, | 514 media::GuessChannelLayout(number_of_channels), |
| 515 sample_rate, |
| 516 16, |
477 frames_per_buffer); | 517 frames_per_buffer); |
478 media::AudioParameters sink_params( | 518 |
479 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, | 519 const int analysis_frames = sample_rate / 100; // 10 ms chunks. |
480 media::CHANNEL_LAYOUT_MONO, kAudioProcessingSampleRate, 16, | 520 render_fifo_.reset( |
481 kAudioProcessingSampleRate / 100); | 521 new MediaStreamAudioFifo(number_of_channels, |
482 render_converter_.reset( | 522 frames_per_buffer, |
483 new MediaStreamAudioConverter(source_params, sink_params)); | 523 analysis_frames)); |
484 render_data_bus_ = media::AudioBus::Create(number_of_channels, | |
485 frames_per_buffer); | |
486 } | 524 } |
487 | 525 |
488 int MediaStreamAudioProcessor::ProcessData(webrtc::AudioFrame* audio_frame, | 526 int MediaStreamAudioProcessor::ProcessData(const float* const* process_ptrs, |
| 527 int process_frames, |
489 base::TimeDelta capture_delay, | 528 base::TimeDelta capture_delay, |
490 int volume, | 529 int volume, |
491 bool key_pressed) { | 530 bool key_pressed, |
| 531 float* const* output_ptrs) { |
| 532 DCHECK(audio_processing_); |
492 DCHECK(capture_thread_checker_.CalledOnValidThread()); | 533 DCHECK(capture_thread_checker_.CalledOnValidThread()); |
493 if (!audio_processing_) | |
494 return 0; | |
495 | 534 |
496 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessData"); | 535 TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessData"); |
497 DCHECK_EQ(audio_processing_->input_sample_rate_hz(), | |
498 capture_converter_->sink_parameters().sample_rate()); | |
499 DCHECK_EQ(audio_processing_->num_input_channels(), | |
500 capture_converter_->sink_parameters().channels()); | |
501 DCHECK_EQ(audio_processing_->num_output_channels(), | |
502 capture_converter_->sink_parameters().channels()); | |
503 | 536 |
504 base::subtle::Atomic32 render_delay_ms = | 537 base::subtle::Atomic32 render_delay_ms = |
505 base::subtle::Acquire_Load(&render_delay_ms_); | 538 base::subtle::Acquire_Load(&render_delay_ms_); |
506 int64 capture_delay_ms = capture_delay.InMilliseconds(); | 539 int64 capture_delay_ms = capture_delay.InMilliseconds(); |
507 DCHECK_LT(capture_delay_ms, | 540 DCHECK_LT(capture_delay_ms, |
508 std::numeric_limits<base::subtle::Atomic32>::max()); | 541 std::numeric_limits<base::subtle::Atomic32>::max()); |
509 int total_delay_ms = capture_delay_ms + render_delay_ms; | 542 int total_delay_ms = capture_delay_ms + render_delay_ms; |
510 if (total_delay_ms > 300) { | 543 if (total_delay_ms > 300) { |
511 LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms | 544 LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms |
512 << "ms; render delay: " << render_delay_ms << "ms"; | 545 << "ms; render delay: " << render_delay_ms << "ms"; |
513 } | 546 } |
514 | 547 |
515 audio_processing_->set_stream_delay_ms(total_delay_ms); | 548 webrtc::AudioProcessing* ap = audio_processing_.get(); |
| 549 ap->set_stream_delay_ms(total_delay_ms); |
516 | 550 |
517 DCHECK_LE(volume, WebRtcAudioDeviceImpl::kMaxVolumeLevel); | 551 DCHECK_LE(volume, WebRtcAudioDeviceImpl::kMaxVolumeLevel); |
518 webrtc::GainControl* agc = audio_processing_->gain_control(); | 552 webrtc::GainControl* agc = ap->gain_control(); |
519 int err = agc->set_stream_analog_level(volume); | 553 int err = agc->set_stream_analog_level(volume); |
520 DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; | 554 DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err; |
521 | 555 |
522 audio_processing_->set_stream_key_pressed(key_pressed); | 556 ap->set_stream_key_pressed(key_pressed); |
523 | 557 |
524 err = audio_processing_->ProcessStream(audio_frame); | 558 err = ap->ProcessStream(process_ptrs, |
| 559 process_frames, |
| 560 input_format_.sample_rate(), |
| 561 MapLayout(input_format_.channel_layout()), |
| 562 output_format_.sample_rate(), |
| 563 MapLayout(output_format_.channel_layout()), |
| 564 output_ptrs); |
525 DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; | 565 DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; |
526 | 566 |
527 if (typing_detector_ && | 567 if (typing_detector_) { |
528 audio_frame->vad_activity_ != webrtc::AudioFrame::kVadUnknown) { | 568 webrtc::VoiceDetection* vad = ap->voice_detection(); |
529 bool vad_active = | 569 DCHECK(vad->is_enabled()); |
530 (audio_frame->vad_activity_ == webrtc::AudioFrame::kVadActive); | 570 bool detected = typing_detector_->Process(key_pressed, |
531 bool typing_detected = typing_detector_->Process(key_pressed, vad_active); | 571 vad->stream_has_voice()); |
532 base::subtle::Release_Store(&typing_detected_, typing_detected); | 572 base::subtle::Release_Store(&typing_detected_, detected); |
533 } | 573 } |
534 | 574 |
535 // Return 0 if the volume has not been changed, otherwise return the new | 575 // Return 0 if the volume hasn't been changed, and otherwise the new volume. |
536 // volume. | |
537 return (agc->stream_analog_level() == volume) ? | 576 return (agc->stream_analog_level() == volume) ? |
538 0 : agc->stream_analog_level(); | 577 0 : agc->stream_analog_level(); |
539 } | 578 } |
540 | 579 |
541 } // namespace content | 580 } // namespace content |
OLD | NEW |