OLD | NEW |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media_recorder/audio_track_recorder.h" | 5 #include "content/renderer/media_recorder/audio_track_recorder.h" |
6 | 6 |
7 #include <stdint.h> | 7 #include <stdint.h> |
8 #include <utility> | 8 #include <utility> |
9 | 9 |
10 #include "base/bind.h" | 10 #include "base/bind.h" |
11 #include "base/macros.h" | 11 #include "base/macros.h" |
12 #include "base/stl_util.h" | 12 #include "base/stl_util.h" |
13 #include "content/renderer/media/media_stream_audio_track.h" | 13 #include "content/renderer/media/media_stream_audio_track.h" |
14 #include "media/base/audio_bus.h" | 14 #include "media/base/audio_bus.h" |
15 #include "media/base/audio_converter.h" | 15 #include "media/base/audio_converter.h" |
| 16 #include "media/base/audio_fifo.h" |
16 #include "media/base/audio_parameters.h" | 17 #include "media/base/audio_parameters.h" |
17 #include "media/base/audio_sample_types.h" | 18 #include "media/base/audio_sample_types.h" |
18 #include "media/base/bind_to_current_loop.h" | 19 #include "media/base/bind_to_current_loop.h" |
19 #include "third_party/opus/src/include/opus.h" | 20 #include "third_party/opus/src/include/opus.h" |
20 | 21 |
21 // Note that this code follows the Chrome media convention of defining a "frame" | 22 // Note that this code follows the Chrome media convention of defining a "frame" |
22 // as "one multi-channel sample" as opposed to another common definition meaning | 23 // as "one multi-channel sample" as opposed to another common definition meaning |
23 // "a chunk of samples". Here this second definition of "frame" is called a | 24 // "a chunk of samples". Here this second definition of "frame" is called a |
24 // "buffer"; so what might be called "frame duration" is instead "buffer | 25 // "buffer"; so what might be called "frame duration" is instead "buffer |
25 // duration", and so on. | 26 // duration", and so on. |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
119 base::ThreadChecker encoder_thread_checker_; | 120 base::ThreadChecker encoder_thread_checker_; |
120 | 121 |
121 // Track Audio (ingress) and Opus encoder input parameters, respectively. They | 122 // Track Audio (ingress) and Opus encoder input parameters, respectively. They |
122 // only differ in their sample_rate() and frames_per_buffer(): output is | 123 // only differ in their sample_rate() and frames_per_buffer(): output is |
123 // 48ksamples/s and 2880, respectively. | 124 // 48ksamples/s and 2880, respectively. |
124 media::AudioParameters input_params_; | 125 media::AudioParameters input_params_; |
125 media::AudioParameters output_params_; | 126 media::AudioParameters output_params_; |
126 | 127 |
127 // Sampling rate adapter between an OpusEncoder supported and the provided. | 128 // Sampling rate adapter between an OpusEncoder supported and the provided. |
128 std::unique_ptr<media::AudioConverter> converter_; | 129 std::unique_ptr<media::AudioConverter> converter_; |
129 std::deque<std::unique_ptr<media::AudioBus>> audio_bus_queue_; | 130 std::unique_ptr<media::AudioFifo> fifo_; |
130 | 131 |
131 // Buffer for passing AudioBus data to OpusEncoder. | 132 // Buffer for passing AudioBus data to OpusEncoder. |
132 std::unique_ptr<float[]> buffer_; | 133 std::unique_ptr<float[]> buffer_; |
133 | 134 |
134 // While |paused_|, AudioBuses are not encoded. | 135 // While |paused_|, AudioBuses are not encoded. |
135 bool paused_; | 136 bool paused_; |
136 | 137 |
137 int frames_queued_; | |
138 | |
139 OpusEncoder* opus_encoder_; | 138 OpusEncoder* opus_encoder_; |
140 | 139 |
141 DISALLOW_COPY_AND_ASSIGN(AudioEncoder); | 140 DISALLOW_COPY_AND_ASSIGN(AudioEncoder); |
142 }; | 141 }; |
143 | 142 |
144 AudioTrackRecorder::AudioEncoder::AudioEncoder( | 143 AudioTrackRecorder::AudioEncoder::AudioEncoder( |
145 const OnEncodedAudioCB& on_encoded_audio_cb, | 144 const OnEncodedAudioCB& on_encoded_audio_cb, |
146 int32_t bits_per_second) | 145 int32_t bits_per_second) |
147 : on_encoded_audio_cb_(on_encoded_audio_cb), | 146 : on_encoded_audio_cb_(on_encoded_audio_cb), |
148 bits_per_second_(bits_per_second), | 147 bits_per_second_(bits_per_second), |
149 paused_(false), | 148 paused_(false), |
150 frames_queued_(0), | |
151 opus_encoder_(nullptr) { | 149 opus_encoder_(nullptr) { |
152 // AudioEncoder is constructed on the thread that ATR lives on, but should | 150 // AudioEncoder is constructed on the thread that ATR lives on, but should |
153 // operate only on the encoder thread after that. Reset | 151 // operate only on the encoder thread after that. Reset |
154 // |encoder_thread_checker_| here, as the next call to CalledOnValidThread() | 152 // |encoder_thread_checker_| here, as the next call to CalledOnValidThread() |
155 // will be from the encoder thread. | 153 // will be from the encoder thread. |
156 encoder_thread_checker_.DetachFromThread(); | 154 encoder_thread_checker_.DetachFromThread(); |
157 } | 155 } |
158 | 156 |
159 AudioTrackRecorder::AudioEncoder::~AudioEncoder() { | 157 AudioTrackRecorder::AudioEncoder::~AudioEncoder() { |
160 // We don't DCHECK that we're on the encoder thread here, as it should have | 158 // We don't DCHECK that we're on the encoder thread here, as it should have |
161 // already been deleted at this point. | 159 // already been deleted at this point. |
162 DestroyExistingOpusEncoder(); | 160 DestroyExistingOpusEncoder(); |
163 } | 161 } |
164 | 162 |
165 void AudioTrackRecorder::AudioEncoder::OnSetFormat( | 163 void AudioTrackRecorder::AudioEncoder::OnSetFormat( |
166 const media::AudioParameters& input_params) { | 164 const media::AudioParameters& input_params) { |
167 DVLOG(1) << __func__; | 165 DVLOG(1) << __func__; |
168 DCHECK(encoder_thread_checker_.CalledOnValidThread()); | 166 DCHECK(encoder_thread_checker_.CalledOnValidThread()); |
169 if (input_params_.Equals(input_params)) | 167 if (input_params_.Equals(input_params)) |
170 return; | 168 return; |
171 | 169 |
172 DestroyExistingOpusEncoder(); | 170 DestroyExistingOpusEncoder(); |
173 | 171 |
174 if (!input_params.IsValid()) { | 172 if (!input_params.IsValid()) { |
175 DLOG(ERROR) << "Invalid params: " << input_params.AsHumanReadableString(); | 173 DLOG(ERROR) << "Invalid params: " << input_params.AsHumanReadableString(); |
176 return; | 174 return; |
177 } | 175 } |
178 | |
179 input_params_ = input_params; | 176 input_params_ = input_params; |
180 input_params_.set_frames_per_buffer(input_params_.sample_rate() * | 177 input_params_.set_frames_per_buffer(input_params_.sample_rate() * |
181 kOpusPreferredBufferDurationMs / | 178 kOpusPreferredBufferDurationMs / |
182 base::Time::kMillisecondsPerSecond); | 179 base::Time::kMillisecondsPerSecond); |
183 | 180 |
184 // third_party/libopus supports up to 2 channels (see implementation of | 181 // third_party/libopus supports up to 2 channels (see implementation of |
185 // opus_encoder_create()): force |output_params_| to at most those. | 182 // opus_encoder_create()): force |output_params_| to at most those. |
186 output_params_ = media::AudioParameters( | 183 output_params_ = media::AudioParameters( |
187 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, | 184 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, |
188 media::GuessChannelLayout(std::min(input_params_.channels(), 2)), | 185 media::GuessChannelLayout(std::min(input_params_.channels(), 2)), |
189 kOpusPreferredSamplingRate, | 186 kOpusPreferredSamplingRate, |
190 input_params_.bits_per_sample(), | 187 input_params_.bits_per_sample(), |
191 kOpusPreferredFramesPerBuffer); | 188 kOpusPreferredFramesPerBuffer); |
192 DVLOG(1) << "|input_params_|:" << input_params_.AsHumanReadableString() | 189 DVLOG(1) << "|input_params_|:" << input_params_.AsHumanReadableString() |
193 << " -->|output_params_|:" << output_params_.AsHumanReadableString(); | 190 << " -->|output_params_|:" << output_params_.AsHumanReadableString(); |
194 | 191 |
195 converter_.reset(new media::AudioConverter(input_params_, output_params_, | 192 converter_.reset(new media::AudioConverter(input_params_, output_params_, |
196 false /* disable_fifo */)); | 193 false /* disable_fifo */)); |
197 converter_->AddInput(this); | 194 converter_->AddInput(this); |
198 converter_->PrimeWithSilence(); | 195 converter_->PrimeWithSilence(); |
199 | 196 |
200 frames_queued_ = 0; | 197 fifo_.reset(new media::AudioFifo( |
201 audio_bus_queue_.clear(); | 198 input_params_.channels(), |
| 199 kMaxNumberOfFifoBuffers * input_params_.frames_per_buffer())); |
202 | 200 |
203 buffer_.reset(new float[output_params_.channels() * | 201 buffer_.reset(new float[output_params_.channels() * |
204 output_params_.frames_per_buffer()]); | 202 output_params_.frames_per_buffer()]); |
205 | 203 |
206 // Initialize OpusEncoder. | 204 // Initialize OpusEncoder. |
207 int opus_result; | 205 int opus_result; |
208 opus_encoder_ = opus_encoder_create(output_params_.sample_rate(), | 206 opus_encoder_ = opus_encoder_create(output_params_.sample_rate(), |
209 output_params_.channels(), | 207 output_params_.channels(), |
210 OPUS_APPLICATION_AUDIO, | 208 OPUS_APPLICATION_AUDIO, |
211 &opus_result); | 209 &opus_result); |
(...skipping 20 matching lines...) Expand all Loading... |
232 std::unique_ptr<media::AudioBus> input_bus, | 230 std::unique_ptr<media::AudioBus> input_bus, |
233 const base::TimeTicks& capture_time) { | 231 const base::TimeTicks& capture_time) { |
234 DVLOG(3) << __func__ << ", #frames " << input_bus->frames(); | 232 DVLOG(3) << __func__ << ", #frames " << input_bus->frames(); |
235 DCHECK(encoder_thread_checker_.CalledOnValidThread()); | 233 DCHECK(encoder_thread_checker_.CalledOnValidThread()); |
236 DCHECK_EQ(input_bus->channels(), input_params_.channels()); | 234 DCHECK_EQ(input_bus->channels(), input_params_.channels()); |
237 DCHECK(!capture_time.is_null()); | 235 DCHECK(!capture_time.is_null()); |
238 DCHECK(converter_); | 236 DCHECK(converter_); |
239 | 237 |
240 if (!is_initialized() || paused_) | 238 if (!is_initialized() || paused_) |
241 return; | 239 return; |
242 | 240 // TODO(mcasas): Consider using a std::deque<std::unique_ptr<AudioBus>> |
243 frames_queued_ += input_bus->frames(); | 241 // instead of |
244 audio_bus_queue_.push_back(std::move(input_bus)); | 242 // an AudioFifo, to avoid copying data needlessly since we know the sizes of |
245 | 243 // both input and output and they are multiples. |
246 const int max_frame_limit = | 244 fifo_->Push(input_bus.get()); |
247 kMaxNumberOfFifoBuffers * input_params_.frames_per_buffer(); | |
248 while (frames_queued_ > max_frame_limit) { | |
249 frames_queued_ -= audio_bus_queue_.front()->frames(); | |
250 audio_bus_queue_.pop_front(); | |
251 } | |
252 | 245 |
253 // Wait to have enough |input_bus|s to guarantee a satisfactory conversion. | 246 // Wait to have enough |input_bus|s to guarantee a satisfactory conversion. |
254 while (frames_queued_ >= input_params_.frames_per_buffer()) { | 247 while (fifo_->frames() >= input_params_.frames_per_buffer()) { |
255 std::unique_ptr<media::AudioBus> audio_bus = media::AudioBus::Create( | 248 std::unique_ptr<media::AudioBus> audio_bus = media::AudioBus::Create( |
256 output_params_.channels(), kOpusPreferredFramesPerBuffer); | 249 output_params_.channels(), kOpusPreferredFramesPerBuffer); |
257 converter_->Convert(audio_bus.get()); | 250 converter_->Convert(audio_bus.get()); |
258 audio_bus->ToInterleaved<media::Float32SampleTypeTraits>( | 251 audio_bus->ToInterleaved<media::Float32SampleTypeTraits>( |
259 audio_bus->frames(), buffer_.get()); | 252 audio_bus->frames(), buffer_.get()); |
260 | 253 |
261 std::unique_ptr<std::string> encoded_data(new std::string()); | 254 std::unique_ptr<std::string> encoded_data(new std::string()); |
262 if (DoEncode(opus_encoder_, buffer_.get(), kOpusPreferredFramesPerBuffer, | 255 if (DoEncode(opus_encoder_, buffer_.get(), kOpusPreferredFramesPerBuffer, |
263 encoded_data.get())) { | 256 encoded_data.get())) { |
264 const base::TimeTicks capture_time_of_first_sample = | 257 const base::TimeTicks capture_time_of_first_sample = |
265 capture_time - | 258 capture_time - |
266 base::TimeDelta::FromMicroseconds(frames_queued_ * | 259 base::TimeDelta::FromMicroseconds(fifo_->frames() * |
267 base::Time::kMicrosecondsPerSecond / | 260 base::Time::kMicrosecondsPerSecond / |
268 input_params_.sample_rate()); | 261 input_params_.sample_rate()); |
269 on_encoded_audio_cb_.Run(output_params_, std::move(encoded_data), | 262 on_encoded_audio_cb_.Run(output_params_, std::move(encoded_data), |
270 capture_time_of_first_sample); | 263 capture_time_of_first_sample); |
271 } | 264 } |
272 } | 265 } |
273 } | 266 } |
274 | 267 |
275 double AudioTrackRecorder::AudioEncoder::ProvideInput( | 268 double AudioTrackRecorder::AudioEncoder::ProvideInput( |
276 media::AudioBus* audio_bus, | 269 media::AudioBus* audio_bus, |
277 uint32_t frames_delayed) { | 270 uint32_t frames_delayed) { |
278 if (audio_bus_queue_.empty()) | 271 fifo_->Consume(audio_bus, 0, audio_bus->frames()); |
279 return 0.0; | |
280 | |
281 frames_queued_ -= audio_bus->frames(); | |
282 audio_bus_queue_.front()->CopyTo(audio_bus); | |
283 audio_bus_queue_.pop_front(); | |
284 return 1.0; // Return volume greater than zero to indicate we have more data. | 272 return 1.0; // Return volume greater than zero to indicate we have more data. |
285 } | 273 } |
286 | 274 |
287 void AudioTrackRecorder::AudioEncoder::DestroyExistingOpusEncoder() { | 275 void AudioTrackRecorder::AudioEncoder::DestroyExistingOpusEncoder() { |
288 // We don't DCHECK that we're on the encoder thread here, as this could be | 276 // We don't DCHECK that we're on the encoder thread here, as this could be |
289 // called from the dtor (main thread) or from OnSetFormat() (encoder thread). | 277 // called from the dtor (main thread) or from OnSetFormat() (encoder thread). |
290 if (opus_encoder_) { | 278 if (opus_encoder_) { |
291 opus_encoder_destroy(opus_encoder_); | 279 opus_encoder_destroy(opus_encoder_); |
292 opus_encoder_ = nullptr; | 280 opus_encoder_ = nullptr; |
293 } | 281 } |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
349 } | 337 } |
350 | 338 |
351 void AudioTrackRecorder::Resume() { | 339 void AudioTrackRecorder::Resume() { |
352 DCHECK(main_render_thread_checker_.CalledOnValidThread()); | 340 DCHECK(main_render_thread_checker_.CalledOnValidThread()); |
353 DCHECK(encoder_); | 341 DCHECK(encoder_); |
354 encoder_thread_.task_runner()->PostTask( | 342 encoder_thread_.task_runner()->PostTask( |
355 FROM_HERE, base::Bind(&AudioEncoder::set_paused, encoder_, false)); | 343 FROM_HERE, base::Bind(&AudioEncoder::set_paused, encoder_, false)); |
356 } | 344 } |
357 | 345 |
358 } // namespace content | 346 } // namespace content |
OLD | NEW |