| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "content/renderer/media/webrtc/processed_local_audio_source.h" | |
| 6 | |
| 7 #include "base/logging.h" | |
| 8 #include "base/metrics/histogram.h" | |
| 9 #include "base/strings/stringprintf.h" | |
| 10 #include "content/renderer/media/audio_device_factory.h" | |
| 11 #include "content/renderer/media/media_stream_audio_processor_options.h" | |
| 12 #include "content/renderer/media/media_stream_constraints_util.h" | |
| 13 #include "content/renderer/media/webrtc/peer_connection_dependency_factory.h" | |
| 14 #include "content/renderer/media/webrtc_audio_device_impl.h" | |
| 15 #include "content/renderer/media/webrtc_logging.h" | |
| 16 #include "content/renderer/render_frame_impl.h" | |
| 17 #include "media/audio/sample_rates.h" | |
| 18 #include "media/base/channel_layout.h" | |
| 19 #include "third_party/webrtc/api/mediaconstraintsinterface.h" | |
| 20 #include "third_party/webrtc/media/base/mediachannel.h" | |
| 21 | |
| 22 namespace content { | |
| 23 | |
| 24 namespace { | |
| 25 // Used as an identifier for ProcessedLocalAudioSource::From(). | |
| 26 void* const kClassIdentifier = const_cast<void**>(&kClassIdentifier); | |
| 27 } // namespace | |
| 28 | |
| 29 ProcessedLocalAudioSource::ProcessedLocalAudioSource( | |
| 30 int consumer_render_frame_id, | |
| 31 const StreamDeviceInfo& device_info, | |
| 32 PeerConnectionDependencyFactory* factory) | |
| 33 : MediaStreamAudioSource(true /* is_local_source */), | |
| 34 consumer_render_frame_id_(consumer_render_frame_id), | |
| 35 pc_factory_(factory), | |
| 36 volume_(0), | |
| 37 allow_invalid_render_frame_id_for_testing_(false) { | |
| 38 DCHECK(pc_factory_); | |
| 39 DVLOG(1) << "ProcessedLocalAudioSource::ProcessedLocalAudioSource()"; | |
| 40 MediaStreamSource::SetDeviceInfo(device_info); | |
| 41 } | |
| 42 | |
| 43 ProcessedLocalAudioSource::~ProcessedLocalAudioSource() { | |
| 44 DVLOG(1) << "ProcessedLocalAudioSource::~ProcessedLocalAudioSource()"; | |
| 45 EnsureSourceIsStopped(); | |
| 46 } | |
| 47 | |
| 48 // static | |
| 49 ProcessedLocalAudioSource* ProcessedLocalAudioSource::From( | |
| 50 MediaStreamAudioSource* source) { | |
| 51 if (source && source->GetClassIdentifier() == kClassIdentifier) | |
| 52 return static_cast<ProcessedLocalAudioSource*>(source); | |
| 53 return nullptr; | |
| 54 } | |
| 55 | |
| 56 void ProcessedLocalAudioSource::SetSourceConstraints( | |
| 57 const blink::WebMediaConstraints& constraints) { | |
| 58 DCHECK(thread_checker_.CalledOnValidThread()); | |
| 59 DCHECK(!constraints.isNull()); | |
| 60 DCHECK(!source_); | |
| 61 constraints_ = constraints; | |
| 62 } | |
| 63 | |
| 64 void* ProcessedLocalAudioSource::GetClassIdentifier() const { | |
| 65 return kClassIdentifier; | |
| 66 } | |
| 67 | |
| 68 bool ProcessedLocalAudioSource::EnsureSourceIsStarted() { | |
| 69 DCHECK(thread_checker_.CalledOnValidThread()); | |
| 70 | |
| 71 if (source_) | |
| 72 return true; | |
| 73 | |
| 74 // Sanity-check that the consuming RenderFrame still exists. This is required | |
| 75 // to initialize the audio source. | |
| 76 if (!allow_invalid_render_frame_id_for_testing_ && | |
| 77 !RenderFrameImpl::FromRoutingID(consumer_render_frame_id_)) { | |
| 78 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails " | |
| 79 " because the render frame does not exist."); | |
| 80 return false; | |
| 81 } | |
| 82 | |
| 83 WebRtcLogMessage(base::StringPrintf( | |
| 84 "ProcessedLocalAudioSource::EnsureSourceIsStarted. render_frame_id=%d" | |
| 85 ", channel_layout=%d, sample_rate=%d, buffer_size=%d" | |
| 86 ", session_id=%d, paired_output_sample_rate=%d" | |
| 87 ", paired_output_frames_per_buffer=%d, effects=%d. ", | |
| 88 consumer_render_frame_id_, device_info().device.input.channel_layout, | |
| 89 device_info().device.input.sample_rate, | |
| 90 device_info().device.input.frames_per_buffer, device_info().session_id, | |
| 91 device_info().device.matched_output.sample_rate, | |
| 92 device_info().device.matched_output.frames_per_buffer, | |
| 93 device_info().device.input.effects)); | |
| 94 | |
| 95 // Sanity-check that the constraints, plus the additional input effects are | |
| 96 // valid when combined. | |
| 97 const MediaAudioConstraints audio_constraints( | |
| 98 constraints_, device_info().device.input.effects); | |
| 99 if (!audio_constraints.IsValid()) { | |
| 100 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails " | |
| 101 " because MediaAudioConstraints are not valid."); | |
| 102 return false; | |
| 103 } | |
| 104 | |
| 105 // Build an AudioOptions by applying relevant constraints to it, and then use | |
| 106 // it to create a webrtc::AudioSourceInterface instance. | |
| 107 cricket::AudioOptions rtc_options; | |
| 108 rtc_options.echo_cancellation = ConstraintToOptional( | |
| 109 constraints_, &blink::WebMediaTrackConstraintSet::echoCancellation); | |
| 110 rtc_options.delay_agnostic_aec = ConstraintToOptional( | |
| 111 constraints_, &blink::WebMediaTrackConstraintSet::googDAEchoCancellation); | |
| 112 rtc_options.auto_gain_control = ConstraintToOptional( | |
| 113 constraints_, &blink::WebMediaTrackConstraintSet::googAutoGainControl); | |
| 114 rtc_options.experimental_agc = ConstraintToOptional( | |
| 115 constraints_, | |
| 116 &blink::WebMediaTrackConstraintSet::googExperimentalAutoGainControl); | |
| 117 rtc_options.noise_suppression = ConstraintToOptional( | |
| 118 constraints_, &blink::WebMediaTrackConstraintSet::googNoiseSuppression); | |
| 119 rtc_options.experimental_ns = ConstraintToOptional( | |
| 120 constraints_, | |
| 121 &blink::WebMediaTrackConstraintSet::googExperimentalNoiseSuppression); | |
| 122 rtc_options.highpass_filter = ConstraintToOptional( | |
| 123 constraints_, &blink::WebMediaTrackConstraintSet::googHighpassFilter); | |
| 124 rtc_options.typing_detection = ConstraintToOptional( | |
| 125 constraints_, | |
| 126 &blink::WebMediaTrackConstraintSet::googTypingNoiseDetection); | |
| 127 rtc_options.stereo_swapping = ConstraintToOptional( | |
| 128 constraints_, &blink::WebMediaTrackConstraintSet::googAudioMirroring); | |
| 129 MediaAudioConstraints::ApplyFixedAudioConstraints(&rtc_options); | |
| 130 if (device_info().device.input.effects & | |
| 131 media::AudioParameters::ECHO_CANCELLER) { | |
| 132 // TODO(hta): Figure out if we should be looking at echoCancellation. | |
| 133 // Previous code had googEchoCancellation only. | |
| 134 const blink::BooleanConstraint& echoCancellation = | |
| 135 constraints_.basic().googEchoCancellation; | |
| 136 if (echoCancellation.hasExact() && !echoCancellation.exact()) { | |
| 137 StreamDeviceInfo modified_device_info(device_info()); | |
| 138 modified_device_info.device.input.effects &= | |
| 139 ~media::AudioParameters::ECHO_CANCELLER; | |
| 140 SetDeviceInfo(modified_device_info); | |
| 141 } | |
| 142 rtc_options.echo_cancellation = rtc::Optional<bool>(false); | |
| 143 } | |
| 144 rtc_source_ = pc_factory_->CreateLocalAudioSource(rtc_options); | |
| 145 if (rtc_source_->state() != webrtc::MediaSourceInterface::kLive) { | |
| 146 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails " | |
| 147 " because the rtc LocalAudioSource is not live."); | |
| 148 return false; | |
| 149 } | |
| 150 | |
| 151 // Create the MediaStreamAudioProcessor, bound to the WebRTC audio device | |
| 152 // module. | |
| 153 WebRtcAudioDeviceImpl* const rtc_audio_device = | |
| 154 pc_factory_->GetWebRtcAudioDevice(); | |
| 155 if (!rtc_audio_device) { | |
| 156 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails " | |
| 157 " because there is no WebRtcAudioDeviceImpl instance."); | |
| 158 return false; | |
| 159 } | |
| 160 audio_processor_ = new rtc::RefCountedObject<MediaStreamAudioProcessor>( | |
| 161 constraints_, device_info().device.input, rtc_audio_device); | |
| 162 | |
| 163 // If KEYBOARD_MIC effect is set, change the layout to the corresponding | |
| 164 // layout that includes the keyboard mic. | |
| 165 media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>( | |
| 166 device_info().device.input.channel_layout); | |
| 167 if ((device_info().device.input.effects & | |
| 168 media::AudioParameters::KEYBOARD_MIC) && | |
| 169 audio_constraints.GetGoogExperimentalNoiseSuppression()) { | |
| 170 if (channel_layout == media::CHANNEL_LAYOUT_STEREO) { | |
| 171 channel_layout = media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC; | |
| 172 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due " | |
| 173 << "to KEYBOARD_MIC effect."; | |
| 174 } else { | |
| 175 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout " | |
| 176 << channel_layout; | |
| 177 } | |
| 178 } | |
| 179 | |
| 180 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout; | |
| 181 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout", | |
| 182 channel_layout, media::CHANNEL_LAYOUT_MAX + 1); | |
| 183 | |
| 184 // Verify that the reported input channel configuration is supported. | |
| 185 if (channel_layout != media::CHANNEL_LAYOUT_MONO && | |
| 186 channel_layout != media::CHANNEL_LAYOUT_STEREO && | |
| 187 channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) { | |
| 188 WebRtcLogMessage(base::StringPrintf( | |
| 189 "ProcessedLocalAudioSource::EnsureSourceIsStarted() fails " | |
| 190 " because the input channel layout (%d) is not supported.", | |
| 191 static_cast<int>(channel_layout))); | |
| 192 return false; | |
| 193 } | |
| 194 | |
| 195 DVLOG(1) << "Audio input hardware sample rate: " | |
| 196 << device_info().device.input.sample_rate; | |
| 197 media::AudioSampleRate asr; | |
| 198 if (media::ToAudioSampleRate(device_info().device.input.sample_rate, &asr)) { | |
| 199 UMA_HISTOGRAM_ENUMERATION( | |
| 200 "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1); | |
| 201 } else { | |
| 202 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected", | |
| 203 device_info().device.input.sample_rate); | |
| 204 } | |
| 205 | |
| 206 // Determine the audio format required of the AudioCapturerSource. Then, pass | |
| 207 // that to the |audio_processor_| and set the output format of this | |
| 208 // ProcessedLocalAudioSource to the processor's output format. | |
| 209 media::AudioParameters params( | |
| 210 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, channel_layout, | |
| 211 device_info().device.input.sample_rate, 16, | |
| 212 GetBufferSize(device_info().device.input.sample_rate)); | |
| 213 params.set_effects(device_info().device.input.effects); | |
| 214 DCHECK(params.IsValid()); | |
| 215 audio_processor_->OnCaptureFormatChanged(params); | |
| 216 MediaStreamAudioSource::SetFormat(audio_processor_->OutputFormat()); | |
| 217 | |
| 218 // Start the source. | |
| 219 VLOG(1) << "Starting WebRTC audio source for consumption by render frame " | |
| 220 << consumer_render_frame_id_ << " with input parameters={" | |
| 221 << params.AsHumanReadableString() << "} and output parameters={" | |
| 222 << GetAudioParameters().AsHumanReadableString() << '}'; | |
| 223 source_ = | |
| 224 AudioDeviceFactory::NewAudioCapturerSource(consumer_render_frame_id_); | |
| 225 source_->Initialize(params, this, device_info().session_id); | |
| 226 // We need to set the AGC control before starting the stream. | |
| 227 source_->SetAutomaticGainControl(true); | |
| 228 source_->Start(); | |
| 229 | |
| 230 // Register this source with the WebRtcAudioDeviceImpl. | |
| 231 rtc_audio_device->AddAudioCapturer(this); | |
| 232 | |
| 233 return true; | |
| 234 } | |
| 235 | |
| 236 void ProcessedLocalAudioSource::EnsureSourceIsStopped() { | |
| 237 DCHECK(thread_checker_.CalledOnValidThread()); | |
| 238 | |
| 239 if (!source_) | |
| 240 return; | |
| 241 | |
| 242 if (WebRtcAudioDeviceImpl* rtc_audio_device = | |
| 243 pc_factory_->GetWebRtcAudioDevice()) { | |
| 244 rtc_audio_device->RemoveAudioCapturer(this); | |
| 245 } | |
| 246 | |
| 247 // Note: Stopping the source while holding the |volume_lock_| because the | |
| 248 // SetVolume() method needs to know whether |source_| is valid. | |
| 249 { | |
| 250 base::AutoLock auto_lock(volume_lock_); | |
| 251 source_->Stop(); | |
| 252 source_ = nullptr; | |
| 253 } | |
| 254 | |
| 255 // Stop the audio processor to avoid feeding render data into the processor. | |
| 256 audio_processor_->Stop(); | |
| 257 | |
| 258 VLOG(1) << "Stopped WebRTC audio pipeline for consumption by render frame " | |
| 259 << consumer_render_frame_id_ << '.'; | |
| 260 } | |
| 261 | |
| 262 void ProcessedLocalAudioSource::SetVolume(int volume) { | |
| 263 DVLOG(1) << "ProcessedLocalAudioSource::SetVolume()"; | |
| 264 DCHECK_LE(volume, MaxVolume()); | |
| 265 double normalized_volume = static_cast<double>(volume) / MaxVolume(); | |
| 266 base::AutoLock auto_lock(volume_lock_); | |
| 267 if (source_) | |
| 268 source_->SetVolume(normalized_volume); | |
| 269 } | |
| 270 | |
| 271 int ProcessedLocalAudioSource::Volume() const { | |
| 272 base::AutoLock auto_lock(volume_lock_); | |
| 273 return volume_; | |
| 274 } | |
| 275 | |
| 276 int ProcessedLocalAudioSource::MaxVolume() const { | |
| 277 return WebRtcAudioDeviceImpl::kMaxVolumeLevel; | |
| 278 } | |
| 279 | |
| 280 void ProcessedLocalAudioSource::Capture(const media::AudioBus* audio_bus, | |
| 281 int audio_delay_milliseconds, | |
| 282 double volume, | |
| 283 bool key_pressed) { | |
| 284 #if defined(OS_WIN) || defined(OS_MACOSX) | |
| 285 DCHECK_LE(volume, 1.0); | |
| 286 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD) | |
| 287 // We have a special situation on Linux where the microphone volume can be | |
| 288 // "higher than maximum". The input volume slider in the sound preference | |
| 289 // allows the user to set a scaling that is higher than 100%. It means that | |
| 290 // even if the reported maximum levels is N, the actual microphone level can | |
| 291 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x. | |
| 292 DCHECK_LE(volume, 1.6); | |
| 293 #endif | |
| 294 | |
| 295 // TODO(miu): Plumbing is needed to determine the actual capture timestamp | |
| 296 // of the audio, instead of just snapshotting TimeTicks::Now(), for proper | |
| 297 // audio/video sync. http://crbug.com/335335 | |
| 298 const base::TimeTicks reference_clock_snapshot = base::TimeTicks::Now(); | |
| 299 | |
| 300 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC. | |
| 301 // The volume can be higher than 255 on Linux, and it will be cropped to | |
| 302 // 255 since AGC does not allow values out of range. | |
| 303 int current_volume = static_cast<int>((volume * MaxVolume()) + 0.5); | |
| 304 { | |
| 305 base::AutoLock auto_lock(volume_lock_); | |
| 306 volume_ = current_volume; | |
| 307 } | |
| 308 current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_; | |
| 309 | |
| 310 // Sanity-check the input audio format in debug builds. Then, notify the | |
| 311 // tracks if the format has changed. | |
| 312 // | |
| 313 // Locking is not needed here to read the audio input/output parameters | |
| 314 // because the audio processor format changes only occur while audio capture | |
| 315 // is stopped. | |
| 316 DCHECK(audio_processor_->InputFormat().IsValid()); | |
| 317 DCHECK_EQ(audio_bus->channels(), audio_processor_->InputFormat().channels()); | |
| 318 DCHECK_EQ(audio_bus->frames(), | |
| 319 audio_processor_->InputFormat().frames_per_buffer()); | |
| 320 | |
| 321 // Figure out if the pre-processed data has any energy or not. This | |
| 322 // information will be passed to the level calculator to force it to report | |
| 323 // energy in case the post-processed data is zeroed by the audio processing. | |
| 324 const bool force_report_nonzero_energy = !audio_bus->AreFramesZero(); | |
| 325 | |
| 326 // Push the data to the processor for processing. | |
| 327 audio_processor_->PushCaptureData( | |
| 328 *audio_bus, | |
| 329 base::TimeDelta::FromMilliseconds(audio_delay_milliseconds)); | |
| 330 | |
| 331 // Process and consume the data in the processor until there is not enough | |
| 332 // data in the processor. | |
| 333 media::AudioBus* processed_data = nullptr; | |
| 334 base::TimeDelta processed_data_audio_delay; | |
| 335 int new_volume = 0; | |
| 336 while (audio_processor_->ProcessAndConsumeData( | |
| 337 current_volume, key_pressed, | |
| 338 &processed_data, &processed_data_audio_delay, &new_volume)) { | |
| 339 DCHECK(processed_data); | |
| 340 | |
| 341 level_calculator_.Calculate(*processed_data, force_report_nonzero_energy); | |
| 342 | |
| 343 MediaStreamAudioSource::DeliverDataToTracks( | |
| 344 *processed_data, reference_clock_snapshot - processed_data_audio_delay); | |
| 345 | |
| 346 if (new_volume) { | |
| 347 SetVolume(new_volume); | |
| 348 | |
| 349 // Update the |current_volume| to avoid passing the old volume to AGC. | |
| 350 current_volume = new_volume; | |
| 351 } | |
| 352 } | |
| 353 } | |
| 354 | |
| 355 void ProcessedLocalAudioSource::OnCaptureError(const std::string& message) { | |
| 356 WebRtcLogMessage("ProcessedLocalAudioSource::OnCaptureError: " + message); | |
| 357 } | |
| 358 | |
| 359 media::AudioParameters ProcessedLocalAudioSource::GetInputFormat() const { | |
| 360 return audio_processor_ ? audio_processor_->InputFormat() | |
| 361 : media::AudioParameters(); | |
| 362 } | |
| 363 | |
| 364 int ProcessedLocalAudioSource::GetBufferSize(int sample_rate) const { | |
| 365 DCHECK(thread_checker_.CalledOnValidThread()); | |
| 366 #if defined(OS_ANDROID) | |
| 367 // TODO(henrika): Re-evaluate whether to use same logic as other platforms. | |
| 368 return (2 * sample_rate / 100); | |
| 369 #endif | |
| 370 | |
| 371 // If audio processing is turned on, require 10ms buffers. | |
| 372 if (audio_processor_->has_audio_processing()) | |
| 373 return (sample_rate / 100); | |
| 374 | |
| 375 // If audio processing is off and the native hardware buffer size was | |
| 376 // provided, use it. It can be harmful, in terms of CPU/power consumption, to | |
| 377 // use smaller buffer sizes than the native size (http://crbug.com/362261). | |
| 378 if (int hardware_buffer_size = device_info().device.input.frames_per_buffer) | |
| 379 return hardware_buffer_size; | |
| 380 | |
| 381 // If the buffer size is missing from the StreamDeviceInfo, provide 10ms as a | |
| 382 // fall-back. | |
| 383 // | |
| 384 // TODO(miu): Identify where/why the buffer size might be missing, fix the | |
| 385 // code, and then require it here. | |
| 386 return (sample_rate / 100); | |
| 387 } | |
| 388 | |
| 389 } // namespace content | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1966043006:
Revert of MediaStream audio: Refactor 3 separate "glue" implementations into one. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master