Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(121)

Side by Side Diff: content/renderer/media/webrtc/processed_local_audio_source.cc

Issue 1647773002: MediaStream audio sourcing: Bypass audio processing for non-WebRTC cases. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: NOT FOR REVIEW -- This will be broken-up across multiple CLs. Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "content/renderer/media/webrtc_audio_capturer.h" 5 #include "content/renderer/media/webrtc/processed_local_audio_source.h"
6 6
7 #include "base/bind.h"
8 #include "base/logging.h" 7 #include "base/logging.h"
9 #include "base/macros.h"
10 #include "base/metrics/histogram.h" 8 #include "base/metrics/histogram.h"
11 #include "base/strings/string_util.h"
12 #include "base/strings/stringprintf.h" 9 #include "base/strings/stringprintf.h"
13 #include "build/build_config.h"
14 #include "content/child/child_process.h"
15 #include "content/renderer/media/audio_device_factory.h" 10 #include "content/renderer/media/audio_device_factory.h"
16 #include "content/renderer/media/media_stream_audio_processor.h" 11 #include "content/renderer/media/media_stream_audio_processor.h"
17 #include "content/renderer/media/media_stream_audio_processor_options.h" 12 #include "content/renderer/media/media_stream_audio_processor_options.h"
18 #include "content/renderer/media/media_stream_audio_source.h"
19 #include "content/renderer/media/media_stream_constraints_util.h" 13 #include "content/renderer/media/media_stream_constraints_util.h"
14 #include "content/renderer/media/rtc_media_constraints.h"
15 #include "content/renderer/media/webrtc/processed_local_audio_track.h"
16 #include "content/renderer/media/webrtc/webrtc_local_audio_track_adapter.h"
20 #include "content/renderer/media/webrtc_audio_device_impl.h" 17 #include "content/renderer/media/webrtc_audio_device_impl.h"
21 #include "content/renderer/media/webrtc_local_audio_track.h"
22 #include "content/renderer/media/webrtc_logging.h" 18 #include "content/renderer/media/webrtc_logging.h"
19 #include "content/renderer/render_frame_impl.h"
20 #include "media/audio/audio_input_device.h"
23 #include "media/audio/sample_rates.h" 21 #include "media/audio/sample_rates.h"
22 #include "media/base/channel_layout.h"
23 #include "third_party/webrtc/api/mediaconstraintsinterface.h"
24 24
25 namespace content { 25 namespace content {
26 26
27 namespace { 27 namespace {
28 28
29 // Audio buffer sizes are specified in milliseconds. 29 // Used as an identifier for ProcessedLocalAudioSource::From().
30 const char kAudioLatency[] = "latencyMs"; 30 void* const kClassIdentifier = const_cast<void**>(&kClassIdentifier);
31 const int kMinAudioLatencyMs = 0; 31
32 const int kMaxAudioLatencyMs = 10000; 32 // Map of corresponding media constraints and platform effects.
33 struct {
34 const char* constraint;
35 const media::AudioParameters::PlatformEffectsMask effect;
36 } const kConstraintEffectMap[] = {
37 { webrtc::MediaConstraintsInterface::kGoogEchoCancellation,
38 media::AudioParameters::ECHO_CANCELLER },
39 };
40
41 // If any platform effects are available, check them against the constraints.
42 // Disable effects to match false constraints, but if a constraint is true, set
43 // the constraint to false to later disable the software effect.
44 //
45 // This function may modify both |constraints| and |effects|.
46 void HarmonizeConstraintsAndEffects(RTCMediaConstraints* constraints,
47 int* effects) {
48 if (*effects != media::AudioParameters::NO_EFFECTS) {
49 for (size_t i = 0; i < arraysize(kConstraintEffectMap); ++i) {
50 bool value;
51 size_t is_mandatory = 0;
52 if (!webrtc::FindConstraint(constraints,
53 kConstraintEffectMap[i].constraint,
54 &value,
55 &is_mandatory) || !value) {
56 // If the constraint is false, or does not exist, disable the platform
57 // effect.
58 *effects &= ~kConstraintEffectMap[i].effect;
59 DVLOG(1) << "Disabling platform effect: "
60 << kConstraintEffectMap[i].effect;
61 } else if (*effects & kConstraintEffectMap[i].effect) {
62 // If the constraint is true, leave the platform effect enabled, and
63 // set the constraint to false to later disable the software effect.
64 if (is_mandatory) {
65 constraints->AddMandatory(kConstraintEffectMap[i].constraint,
66 webrtc::MediaConstraintsInterface::kValueFalse, true);
67 } else {
68 constraints->AddOptional(kConstraintEffectMap[i].constraint,
69 webrtc::MediaConstraintsInterface::kValueFalse, true);
70 }
71 DVLOG(1) << "Disabling constraint: "
72 << kConstraintEffectMap[i].constraint;
73 }
74 }
75 }
76 }
33 77
34 // Method to check if any of the data in |audio_source| has energy. 78 // Method to check if any of the data in |audio_source| has energy.
35 bool HasDataEnergy(const media::AudioBus& audio_source) { 79 bool HasDataEnergy(const media::AudioBus& audio_source) {
36 for (int ch = 0; ch < audio_source.channels(); ++ch) { 80 for (int ch = 0; ch < audio_source.channels(); ++ch) {
37 const float* channel_ptr = audio_source.channel(ch); 81 const float* channel_ptr = audio_source.channel(ch);
38 for (int frame = 0; frame < audio_source.frames(); ++frame) { 82 for (int frame = 0; frame < audio_source.frames(); ++frame) {
39 if (channel_ptr[frame] != 0) 83 if (channel_ptr[frame] != 0)
40 return true; 84 return true;
41 } 85 }
42 } 86 }
43 87
44 // All the data is zero. 88 // All the data is zero.
45 return false; 89 return false;
46 } 90 }
47 91
48 } // namespace 92 } // namespace
49 93
50 // Reference counted container of WebRtcLocalAudioTrack delegate. 94 ProcessedLocalAudioSource::ProcessedLocalAudioSource(
51 // TODO(xians): Switch to MediaStreamAudioSinkOwner. 95 int consumer_render_frame_id,
52 class WebRtcAudioCapturer::TrackOwner 96 const StreamDeviceInfo& device_info,
53 : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> { 97 PeerConnectionDependencyFactory* factory)
54 public: 98 : MediaStreamAudioSource(true /* is_local_source */),
55 explicit TrackOwner(WebRtcLocalAudioTrack* track) 99 consumer_render_frame_id_(consumer_render_frame_id),
56 : delegate_(track) {} 100 pc_factory_(factory),
101 exposed_volume_(0),
102 allow_invalid_render_frame_id_for_testing_(false) {
103 DCHECK(pc_factory_);
104 DVLOG(1) << "ProcessedLocalAudioSource::ProcessedLocalAudioSource()";
105 MediaStreamSource::SetDeviceInfo(device_info);
106 }
57 107
58 void Capture(const media::AudioBus& audio_bus, 108 ProcessedLocalAudioSource::~ProcessedLocalAudioSource() {
59 base::TimeTicks estimated_capture_time, 109 DVLOG(1) << "ProcessedLocalAudioSource::~ProcessedLocalAudioSource()";
60 bool force_report_nonzero_energy) { 110 // Superclass will call StopSource() just in case.
61 base::AutoLock lock(lock_); 111 }
62 if (delegate_) { 112
63 delegate_->Capture(audio_bus, 113 // static
64 estimated_capture_time, 114 ProcessedLocalAudioSource* ProcessedLocalAudioSource::From(
65 force_report_nonzero_energy); 115 MediaStreamAudioSource* source) {
66 } 116 if (source && source->GetClassIdentifier() == kClassIdentifier)
117 return static_cast<ProcessedLocalAudioSource*>(source);
118 return nullptr;
119 }
120
121 void ProcessedLocalAudioSource::SetSourceConstraints(
122 const blink::WebMediaConstraints& constraints) {
123 DCHECK(!constraints.isNull());
124 constraints_ = constraints;
125 }
126
127 void* ProcessedLocalAudioSource::GetClassIdentifier() const {
128 return kClassIdentifier;
129 }
130
131 void ProcessedLocalAudioSource::DoStopSource() {
132 DCHECK(thread_checker_.CalledOnValidThread());
133 if (is_stopped_)
134 return;
135
136 // Setting |is_stopped_| while holding the |volume_lock_| because the
137 // SetVolume() method needs to know whether |input_device_| is valid.
138 {
139 base::AutoLock auto_lock(volume_lock_);
140 is_stopped_ = true;
67 } 141 }
68 142
69 void OnSetFormat(const media::AudioParameters& params) { 143 if (input_device_) {
70 base::AutoLock lock(lock_); 144 if (WebRtcAudioDeviceImpl* rtc_audio_device =
71 if (delegate_) 145 pc_factory_->GetWebRtcAudioDevice()) {
72 delegate_->OnSetFormat(params); 146 rtc_audio_device->RemoveAudioCapturer(this);
147 }
148
149 input_device_->Stop();
150
151 // Stop the audio processor to avoid feeding render data into the processor.
152 audio_processor_->Stop();
153
154 VLOG(1) << "Stopped WebRTC audio pipeline for consumption by render frame "
155 << consumer_render_frame_id_ << '.';
156 }
157 }
158
159 scoped_ptr<MediaStreamAudioTrack>
160 ProcessedLocalAudioSource::CreateMediaStreamAudioTrack(
161 const std::string& id) {
162 DCHECK(thread_checker_.CalledOnValidThread());
163
164 ProcessedLocalAudioTrack* const audio_track =
165 new ProcessedLocalAudioTrack(WebRtcLocalAudioTrackAdapter::Create(
166 id, rtc_source_.get(), pc_factory_->GetWebRtcSignalingThread()));
167 audio_track->adapter()->SetAudioProcessor(audio_processor_);
168 audio_track->adapter()->SetReportedLevel(level_calculator_.reported_level());
169 return scoped_ptr<MediaStreamAudioTrack>(audio_track);
170 }
171
172 bool ProcessedLocalAudioSource::EnsureSourceIsStarted() {
173 DCHECK(thread_checker_.CalledOnValidThread());
174
175 if (is_stopped_)
176 return false;
177 if (input_device_)
178 return true;
179
180 // Sanity-check that the consuming RenderFrame still exists. This is
181 // required to initialize the audio source.
182 if (!allow_invalid_render_frame_id_for_testing_ &&
183 !RenderFrameImpl::FromRoutingID(consumer_render_frame_id_)) {
184 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails "
185 " because the render frame does not exist.");
186 StopSource();
187 return false;
73 } 188 }
74 189
75 void SetAudioProcessor( 190 // Using |constraints_| as a basis, apply additional default constraints for
76 const scoped_refptr<MediaStreamAudioProcessor>& processor) { 191 // audio processing and take the |effects| from StreamDeviceInfo into account.
77 base::AutoLock lock(lock_); 192 //
78 if (delegate_) 193 // TODO(miu): Consolidation of logic needed here: There is both a
79 delegate_->SetAudioProcessor(processor); 194 // RTCMediaConstraints and MediaAudioConstraints class, plus the constraints
195 // are being modified both within and outside this module. (This problem was
196 // exposed after a major refactoring.)
197 RTCMediaConstraints rtc_constraints(constraints_);
198 MediaAudioConstraints::ApplyFixedAudioConstraints(&rtc_constraints);
199 StreamDeviceInfo modified_device_info = device_info();
200 HarmonizeConstraintsAndEffects(&rtc_constraints,
201 &modified_device_info.device.input.effects);
202 MediaStreamSource::SetDeviceInfo(modified_device_info);
203 MediaAudioConstraints audio_constraints(
204 constraints_, modified_device_info.device.input.effects);
205 if (!audio_constraints.IsValid()) {
206 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails "
207 " because MediaAudioConstraints are not valid.");
208 StopSource();
209 return false;
80 } 210 }
81 211
82 void Reset() { 212 const MediaStreamDevice::AudioDeviceParameters& source_params =
83 base::AutoLock lock(lock_); 213 modified_device_info.device.input;
84 delegate_ = NULL; 214 const MediaStreamDevice::AudioDeviceParameters& matched_params =
85 } 215 modified_device_info.device.matched_output;
86
87 void Stop() {
88 base::AutoLock lock(lock_);
89 DCHECK(delegate_);
90
91 // This can be reentrant so reset |delegate_| before calling out.
92 WebRtcLocalAudioTrack* temp = delegate_;
93 delegate_ = NULL;
94 temp->Stop();
95 }
96
97 // Wrapper which allows to use std::find_if() when adding and removing
98 // sinks to/from the list.
99 struct TrackWrapper {
100 explicit TrackWrapper(WebRtcLocalAudioTrack* track) : track_(track) {}
101 bool operator()(
102 const scoped_refptr<WebRtcAudioCapturer::TrackOwner>& owner) const {
103 return owner->IsEqual(track_);
104 }
105 WebRtcLocalAudioTrack* track_;
106 };
107
108 protected:
109 virtual ~TrackOwner() {}
110
111 private:
112 friend class base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner>;
113
114 bool IsEqual(const WebRtcLocalAudioTrack* other) const {
115 base::AutoLock lock(lock_);
116 return (other == delegate_);
117 }
118
119 // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
120 WebRtcLocalAudioTrack* delegate_;
121 mutable base::Lock lock_;
122
123 DISALLOW_COPY_AND_ASSIGN(TrackOwner);
124 };
125
126 // static
127 scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
128 int render_frame_id,
129 const StreamDeviceInfo& device_info,
130 const blink::WebMediaConstraints& constraints,
131 WebRtcAudioDeviceImpl* audio_device,
132 MediaStreamAudioSource* audio_source) {
133 scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
134 render_frame_id, device_info, constraints, audio_device, audio_source);
135 if (capturer->Initialize())
136 return capturer;
137
138 return NULL;
139 }
140
141 bool WebRtcAudioCapturer::Initialize() {
142 DCHECK(thread_checker_.CalledOnValidThread());
143 DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
144 WebRtcLogMessage(base::StringPrintf( 216 WebRtcLogMessage(base::StringPrintf(
145 "WAC::Initialize. render_frame_id=%d" 217 "ProcessedLocalAudioSource::EnsureSourceIsStarted. PRELIMINARY "
218 "parameters: render_frame_id=%d"
146 ", channel_layout=%d, sample_rate=%d, buffer_size=%d" 219 ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
147 ", session_id=%d, paired_output_sample_rate=%d" 220 ", session_id=%d, paired_output_sample_rate=%d"
148 ", paired_output_frames_per_buffer=%d, effects=%d. ", 221 ", paired_output_frames_per_buffer=%d, effects=%d.",
149 render_frame_id_, device_info_.device.input.channel_layout, 222 consumer_render_frame_id_, source_params.channel_layout,
150 device_info_.device.input.sample_rate, 223 source_params.sample_rate, source_params.frames_per_buffer,
151 device_info_.device.input.frames_per_buffer, device_info_.session_id, 224 modified_device_info.session_id, matched_params.sample_rate,
152 device_info_.device.matched_output.sample_rate, 225 matched_params.frames_per_buffer, source_params.effects));
153 device_info_.device.matched_output.frames_per_buffer,
154 device_info_.device.input.effects));
155 226
156 if (render_frame_id_ == -1) { 227 // Create the MediaStreamAudioProcessor, bound to the WebRTC audio device
157 // Return true here to allow injecting a new source via 228 // module.
158 // SetCapturerSourceForTesting() at a later state. 229 WebRtcAudioDeviceImpl* const rtc_audio_device =
159 return true; 230 pc_factory_->GetWebRtcAudioDevice();
231 if (!rtc_audio_device) {
232 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails "
233 " because there is no WebRtcAudioDeviceImpl instance.");
234 StopSource();
235 return false;
160 } 236 }
161 237 audio_processor_ = new rtc::RefCountedObject<MediaStreamAudioProcessor>(
162 MediaAudioConstraints audio_constraints(constraints_, 238 constraints_, source_params, rtc_audio_device);
163 device_info_.device.input.effects);
164 if (!audio_constraints.IsValid())
165 return false;
166
167 media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>(
168 device_info_.device.input.channel_layout);
169 239
170 // If KEYBOARD_MIC effect is set, change the layout to the corresponding 240 // If KEYBOARD_MIC effect is set, change the layout to the corresponding
171 // layout that includes the keyboard mic. 241 // layout that includes the keyboard mic.
172 if ((device_info_.device.input.effects & 242 media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>(
243 modified_device_info.device.input.channel_layout);
244 if ((modified_device_info.device.input.effects &
173 media::AudioParameters::KEYBOARD_MIC) && 245 media::AudioParameters::KEYBOARD_MIC) &&
174 audio_constraints.GetProperty( 246 audio_constraints.GetProperty(
175 MediaAudioConstraints::kGoogExperimentalNoiseSuppression)) { 247 MediaAudioConstraints::kGoogExperimentalNoiseSuppression)) {
176 if (channel_layout == media::CHANNEL_LAYOUT_STEREO) { 248 if (channel_layout == media::CHANNEL_LAYOUT_STEREO) {
177 channel_layout = media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC; 249 channel_layout = media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC;
178 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due " 250 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due "
179 << "to KEYBOARD_MIC effect."; 251 << "to KEYBOARD_MIC effect.";
180 } else { 252 } else {
181 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout " 253 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout "
182 << channel_layout; 254 << channel_layout;
183 } 255 }
184 } 256 }
185
186 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout; 257 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout;
187 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout", 258 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
188 channel_layout, media::CHANNEL_LAYOUT_MAX + 1); 259 channel_layout, media::CHANNEL_LAYOUT_MAX + 1);
189 260
190 // Verify that the reported input channel configuration is supported. 261 // Verify that the input channel configuration is supported.
191 if (channel_layout != media::CHANNEL_LAYOUT_MONO && 262 if (channel_layout != media::CHANNEL_LAYOUT_MONO &&
192 channel_layout != media::CHANNEL_LAYOUT_STEREO && 263 channel_layout != media::CHANNEL_LAYOUT_STEREO &&
193 channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) { 264 channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) {
194 DLOG(ERROR) << channel_layout 265 WebRtcLogMessage(base::StringPrintf(
195 << " is not a supported input channel configuration."; 266 "ProcessedLocalAudioSource::EnsureSourceIsStarted() fails "
267 " because the input channel layout (%d) is not supported.",
268 static_cast<int>(channel_layout)));
269 StopSource();
196 return false; 270 return false;
197 } 271 }
198 272
199 DVLOG(1) << "Audio input hardware sample rate: " 273 DVLOG(1) << "Audio input hardware sample rate: "
200 << device_info_.device.input.sample_rate; 274 << modified_device_info.device.input.sample_rate;
201 media::AudioSampleRate asr; 275 media::AudioSampleRate asr;
202 if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) { 276 if (media::ToAudioSampleRate(modified_device_info.device.input.sample_rate,
277 &asr)) {
203 UMA_HISTOGRAM_ENUMERATION( 278 UMA_HISTOGRAM_ENUMERATION(
204 "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1); 279 "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
205 } else { 280 } else {
206 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected", 281 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
207 device_info_.device.input.sample_rate); 282 modified_device_info.device.input.sample_rate);
208 } 283 }
209 284
210 // Initialize the buffer size to zero, which means it wasn't specified. 285 // The buffer size is 20 ms on Android, and 10 ms everywhere else.
211 // If it is out of range, we return it to zero. 286 #if defined(OS_ANDROID)
212 int buffer_size_ms = 0; 287 // TODO(henrika): Tune and adjust buffer size on Android.
213 int buffer_size_samples = 0; 288 const int buffer_size_samples =
214 GetConstraintValueAsInteger(constraints_, kAudioLatency, &buffer_size_ms); 289 modified_device_info.device.input.sample_rate / 50;
215 if (buffer_size_ms < kMinAudioLatencyMs || 290 #else
216 buffer_size_ms > kMaxAudioLatencyMs) { 291 const int buffer_size_samples =
217 DVLOG(1) << "Ignoring out of range buffer size " << buffer_size_ms; 292 modified_device_info.device.input.sample_rate / 100;
218 } else { 293 #endif
219 buffer_size_samples = 294
220 device_info_.device.input.sample_rate * buffer_size_ms / 1000; 295 // Determine the audio format required of the AudioInputDevice. Then, pass
296 // that to the |audio_processor_| and set the output format of this
297 // ProcessedLocalAudioSource to the processor's output format.
298 media::AudioParameters params(
299 media::AudioParameters::AUDIO_PCM_LOW_LATENCY, channel_layout,
300 modified_device_info.device.input.sample_rate, 16, buffer_size_samples);
301 params.set_effects(modified_device_info.device.input.effects);
302 DCHECK(params.IsValid());
303 audio_processor_->OnCaptureFormatChanged(params);
304 MediaStreamAudioSource::SetFormat(audio_processor_->OutputFormat());
305
306 // Start the source.
307 VLOG(1) << "Starting WebRTC audio source for consumption by render frame "
308 << consumer_render_frame_id_ << " with audio parameters={"
309 << GetAudioParameters().AsHumanReadableString() << '}';
310 scoped_refptr<media::AudioInputDevice> device =
311 AudioDeviceFactory::NewInputDevice(consumer_render_frame_id_);
312 device->Initialize(params, this, modified_device_info.session_id);
313 // We need to set the AGC control before starting the stream.
314 device->SetAutomaticGainControl(true);
315 device->Start();
316 input_device_ = device; // Thread-safe assignment.
317
318 // Register this source with the WebRtcAudioDeviceImpl.
319 rtc_audio_device->AddAudioCapturer(this);
320
321 // Creates a LocalAudioSource object which holds audio options.
322 // TODO(xians): The option should apply to the track instead of the source.
323 // TODO(perkj): Move audio constraints parsing to Chrome.
324 // Currently there are a few constraints that are parsed by libjingle and
325 // the state is set to ended if parsing fails.
326 rtc_source_ = pc_factory_->CreateLocalAudioSource(&rtc_constraints);
327 if (rtc_source_->state() != webrtc::MediaSourceInterface::kLive) {
328 WebRtcLogMessage("ProcessedLocalAudioSource::EnsureSourceIsStarted() fails "
329 " because the rtc LocalAudioSource is not live.");
330 StopSource();
331 return false;
221 } 332 }
222 DVLOG_IF(1, buffer_size_samples > 0)
223 << "Custom audio buffer size: " << buffer_size_samples << " samples";
224
225 // Create and configure the default audio capturing source.
226 SetCapturerSourceInternal(
227 AudioDeviceFactory::NewInputDevice(render_frame_id_),
228 channel_layout,
229 device_info_.device.input.sample_rate,
230 buffer_size_samples);
231
232 // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
233 // information from the capturer.
234 if (audio_device_)
235 audio_device_->AddAudioCapturer(this);
236 333
237 return true; 334 return true;
238 } 335 }
239 336
240 WebRtcAudioCapturer::WebRtcAudioCapturer( 337 void ProcessedLocalAudioSource::SetVolume(int volume) {
241 int render_frame_id, 338 DVLOG(1) << "ProcessedLocalAudioSource::SetVolume()";
242 const StreamDeviceInfo& device_info, 339 DCHECK_LE(volume, MaxVolume());
243 const blink::WebMediaConstraints& constraints,
244 WebRtcAudioDeviceImpl* audio_device,
245 MediaStreamAudioSource* audio_source)
246 : constraints_(constraints),
247 audio_processor_(new rtc::RefCountedObject<MediaStreamAudioProcessor>(
248 constraints,
249 device_info.device.input,
250 audio_device)),
251 running_(false),
252 render_frame_id_(render_frame_id),
253 device_info_(device_info),
254 volume_(0),
255 peer_connection_mode_(false),
256 audio_device_(audio_device),
257 audio_source_(audio_source) {
258 DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
259 }
260
261 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
262 DCHECK(thread_checker_.CalledOnValidThread());
263 DCHECK(tracks_.IsEmpty());
264 DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
265 Stop();
266 }
267
268 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
269 DCHECK(track);
270 DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
271 340
272 { 341 {
273 base::AutoLock auto_lock(lock_); 342 base::AutoLock auto_lock(volume_lock_);
274 // Verify that |track| is not already added to the list. 343 if (is_stopped_)
275 DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track))); 344 return;
345 }
276 346
277 // Add with a tag, so we remember to call OnSetFormat() on the new 347 // Assumption: Once |input_device_| is set, it will never change. Thus,
278 // track. 348 // there's no need to hold any locks for the following:
279 scoped_refptr<TrackOwner> track_owner(new TrackOwner(track)); 349 if (input_device_.get()) {
280 tracks_.AddAndTag(track_owner.get()); 350 double normalized_volume = static_cast<double>(volume) / MaxVolume();
351 input_device_->SetVolume(normalized_volume);
281 } 352 }
282 } 353 }
283 354
284 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack* track) { 355 int ProcessedLocalAudioSource::Volume() const {
285 DCHECK(thread_checker_.CalledOnValidThread()); 356 base::AutoLock auto_lock(volume_lock_);
286 DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()"; 357 return exposed_volume_;
287 bool stop_source = false;
288 {
289 base::AutoLock auto_lock(lock_);
290
291 scoped_refptr<TrackOwner> removed_item =
292 tracks_.Remove(TrackOwner::TrackWrapper(track));
293
294 // Clear the delegate to ensure that no more capture callbacks will
295 // be sent to this sink. Also avoids a possible crash which can happen
296 // if this method is called while capturing is active.
297 if (removed_item.get()) {
298 removed_item->Reset();
299 stop_source = tracks_.IsEmpty();
300 }
301 }
302 if (stop_source) {
303 // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
304 // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
305 // we have to call StopSource on the MediaStreamSource. This will call
306 // MediaStreamAudioSource::DoStopSource which in turn call
307 // WebRtcAudioCapturerer::Stop();
308 audio_source_->StopSource();
309 }
310 } 358 }
311 359
312 void WebRtcAudioCapturer::SetCapturerSourceInternal( 360 int ProcessedLocalAudioSource::MaxVolume() const {
313 const scoped_refptr<media::AudioCapturerSource>& source,
314 media::ChannelLayout channel_layout,
315 int sample_rate,
316 int buffer_size) {
317 DCHECK(thread_checker_.CalledOnValidThread());
318 DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
319 << "sample_rate=" << sample_rate << ")";
320 scoped_refptr<media::AudioCapturerSource> old_source;
321 {
322 base::AutoLock auto_lock(lock_);
323 if (source_.get() == source.get())
324 return;
325
326 source_.swap(old_source);
327 source_ = source;
328
329 // Reset the flag to allow starting the new source.
330 running_ = false;
331 }
332
333 DVLOG(1) << "Switching to a new capture source.";
334 if (old_source.get())
335 old_source->Stop();
336
337 // If the buffer size is zero, it has not been specified.
338 // We either default to 10ms, or use the hardware buffer size.
339 if (buffer_size == 0)
340 buffer_size = GetBufferSize(sample_rate);
341
342 // Dispatch the new parameters both to the sink(s) and to the new source,
343 // also apply the new |constraints|.
344 // The idea is to get rid of any dependency of the microphone parameters
345 // which would normally be used by default.
346 // bits_per_sample is always 16 for now.
347 media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
348 channel_layout, sample_rate, 16, buffer_size);
349 params.set_effects(device_info_.device.input.effects);
350
351 {
352 base::AutoLock auto_lock(lock_);
353 // Notify the |audio_processor_| of the new format.
354 audio_processor_->OnCaptureFormatChanged(params);
355
356 // Notify all tracks about the new format.
357 tracks_.TagAll();
358 }
359
360 if (source.get())
361 source->Initialize(params, this, session_id());
362
363 Start();
364 }
365
366 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
367 DCHECK(thread_checker_.CalledOnValidThread());
368 DVLOG(1) << "EnablePeerConnectionMode";
369 // Do nothing if the peer connection mode has been enabled.
370 if (peer_connection_mode_)
371 return;
372
373 peer_connection_mode_ = true;
374 int render_frame_id = -1;
375 media::AudioParameters input_params;
376 {
377 base::AutoLock auto_lock(lock_);
378 // Simply return if there is no existing source or the |render_frame_id_| is
379 // not valid.
380 if (!source_.get() || render_frame_id_ == -1)
381 return;
382
383 render_frame_id = render_frame_id_;
384 input_params = audio_processor_->InputFormat();
385 }
386
387 // Do nothing if the current buffer size is the WebRtc native buffer size.
388 if (GetBufferSize(input_params.sample_rate()) ==
389 input_params.frames_per_buffer()) {
390 return;
391 }
392
393 // Create a new audio stream as source which will open the hardware using
394 // WebRtc native buffer size.
395 SetCapturerSourceInternal(AudioDeviceFactory::NewInputDevice(render_frame_id),
396 input_params.channel_layout(),
397 input_params.sample_rate(),
398 0);
399 }
400
401 void WebRtcAudioCapturer::Start() {
402 DCHECK(thread_checker_.CalledOnValidThread());
403 DVLOG(1) << "WebRtcAudioCapturer::Start()";
404 base::AutoLock auto_lock(lock_);
405 if (running_ || !source_.get())
406 return;
407
408 // Start the data source, i.e., start capturing data from the current source.
409 // We need to set the AGC control before starting the stream.
410 source_->SetAutomaticGainControl(true);
411 source_->Start();
412 running_ = true;
413 }
414
415 void WebRtcAudioCapturer::Stop() {
416 DCHECK(thread_checker_.CalledOnValidThread());
417 DVLOG(1) << "WebRtcAudioCapturer::Stop()";
418 scoped_refptr<media::AudioCapturerSource> source;
419 TrackList::ItemList tracks;
420 {
421 base::AutoLock auto_lock(lock_);
422 if (!running_)
423 return;
424
425 source = source_;
426 tracks = tracks_.Items();
427 tracks_.Clear();
428 running_ = false;
429 }
430
431 // Remove the capturer object from the WebRtcAudioDeviceImpl.
432 if (audio_device_)
433 audio_device_->RemoveAudioCapturer(this);
434
435 for (TrackList::ItemList::const_iterator it = tracks.begin();
436 it != tracks.end();
437 ++it) {
438 (*it)->Stop();
439 }
440
441 if (source.get())
442 source->Stop();
443
444 // Stop the audio processor to avoid feeding render data into the processor.
445 audio_processor_->Stop();
446 }
447
448 void WebRtcAudioCapturer::SetVolume(int volume) {
449 DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
450 DCHECK_LE(volume, MaxVolume());
451 double normalized_volume = static_cast<double>(volume) / MaxVolume();
452 base::AutoLock auto_lock(lock_);
453 if (source_.get())
454 source_->SetVolume(normalized_volume);
455 }
456
457 int WebRtcAudioCapturer::Volume() const {
458 base::AutoLock auto_lock(lock_);
459 return volume_;
460 }
461
462 int WebRtcAudioCapturer::MaxVolume() const {
463 return WebRtcAudioDeviceImpl::kMaxVolumeLevel; 361 return WebRtcAudioDeviceImpl::kMaxVolumeLevel;
464 } 362 }
465 363
466 media::AudioParameters WebRtcAudioCapturer::GetOutputFormat() const { 364 void ProcessedLocalAudioSource::Capture(const media::AudioBus* audio_bus,
467 DCHECK(thread_checker_.CalledOnValidThread()); 365 int audio_delay_milliseconds,
468 return audio_processor_->OutputFormat(); 366 double volume,
469 } 367 bool key_pressed) {
470
471 void WebRtcAudioCapturer::Capture(const media::AudioBus* audio_source,
472 int audio_delay_milliseconds,
473 double volume,
474 bool key_pressed) {
475 // This callback is driven by AudioInputDevice::AudioThreadCallback if
476 // |source_| is AudioInputDevice, otherwise it is driven by client's
477 // CaptureCallback.
478 #if defined(OS_WIN) || defined(OS_MACOSX) 368 #if defined(OS_WIN) || defined(OS_MACOSX)
479 DCHECK_LE(volume, 1.0); 369 DCHECK_LE(volume, 1.0);
480 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD) 370 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
481 // We have a special situation on Linux where the microphone volume can be 371 // We have a special situation on Linux where the microphone volume can be
482 // "higher than maximum". The input volume slider in the sound preference 372 // "higher than maximum". The input volume slider in the sound preference
483 // allows the user to set a scaling that is higher than 100%. It means that 373 // allows the user to set a scaling that is higher than 100%. It means that
484 // even if the reported maximum levels is N, the actual microphone level can 374 // even if the reported maximum levels is N, the actual microphone level can
485 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x. 375 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
486 DCHECK_LE(volume, 1.6); 376 DCHECK_LE(volume, 1.6);
487 #endif 377 #endif
488 378
489 // TODO(miu): Plumbing is needed to determine the actual capture timestamp 379 // TODO(miu): Plumbing is needed to determine the actual capture timestamp
490 // of the audio, instead of just snapshotting TimeTicks::Now(), for proper 380 // of the audio, instead of just snapshotting TimeTicks::Now(), for proper
491 // audio/video sync. http://crbug.com/335335 381 // audio/video sync. http://crbug.com/335335
492 const base::TimeTicks reference_clock_snapshot = base::TimeTicks::Now(); 382 const base::TimeTicks reference_clock_snapshot = base::TimeTicks::Now();
493 383
494 TrackList::ItemList tracks; 384 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
495 TrackList::ItemList tracks_to_notify_format; 385 int current_volume = static_cast<int>((volume * MaxVolume()) + 0.5);
496 int current_volume = 0;
497 { 386 {
498 base::AutoLock auto_lock(lock_); 387 base::AutoLock auto_lock(volume_lock_);
499 if (!running_) 388 exposed_volume_ = current_volume;
500 return;
501
502 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
503 // The volume can be higher than 255 on Linux, and it will be cropped to
504 // 255 since AGC does not allow values out of range.
505 volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
506 current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
507 tracks = tracks_.Items();
508 tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
509 } 389 }
390 // The volume can be higher than 255 on Linux, and it will be cropped to
391 // 255 since AGC does not allow values out of range.
392 current_volume = current_volume > MaxVolume() ? MaxVolume() : current_volume;
510 393
511 DCHECK(audio_processor_->InputFormat().IsValid()); 394 DCHECK(audio_processor_->InputFormat().IsValid());
512 DCHECK_EQ(audio_source->channels(), 395 DCHECK_EQ(audio_bus->channels(), audio_processor_->InputFormat().channels());
513 audio_processor_->InputFormat().channels()); 396 DCHECK_EQ(audio_bus->frames(),
514 DCHECK_EQ(audio_source->frames(),
515 audio_processor_->InputFormat().frames_per_buffer()); 397 audio_processor_->InputFormat().frames_per_buffer());
516 398
517 // Notify the tracks on when the format changes. This will do nothing if 399 // Figure out if the pre-processed data has any energy or not.
518 // |tracks_to_notify_format| is empty. 400 const bool force_report_nonzero_energy = HasDataEnergy(*audio_bus);
519 const media::AudioParameters& output_params =
520 audio_processor_->OutputFormat();
521 for (const auto& track : tracks_to_notify_format) {
522 track->OnSetFormat(output_params);
523 track->SetAudioProcessor(audio_processor_);
524 }
525
526 // Figure out if the pre-processed data has any energy or not, the
527 // information will be passed to the track to force the calculator
528 // to report energy in case the post-processed data is zeroed by the audio
529 // processing.
530 const bool force_report_nonzero_energy = HasDataEnergy(*audio_source);
531 401
532 // Push the data to the processor for processing. 402 // Push the data to the processor for processing.
533 audio_processor_->PushCaptureData( 403 audio_processor_->PushCaptureData(
534 *audio_source, 404 *audio_bus,
535 base::TimeDelta::FromMilliseconds(audio_delay_milliseconds)); 405 base::TimeDelta::FromMilliseconds(audio_delay_milliseconds));
536 406
537 // Process and consume the data in the processor until there is not enough 407 // Process and consume the data in the processor until there is not enough
538 // data in the processor. 408 // data in the processor.
539 media::AudioBus* processed_data = nullptr; 409 media::AudioBus* processed_data = nullptr;
540 base::TimeDelta processed_data_audio_delay; 410 base::TimeDelta processed_data_audio_delay;
541 int new_volume = 0; 411 int new_volume = 0;
542 while (audio_processor_->ProcessAndConsumeData( 412 while (audio_processor_->ProcessAndConsumeData(
543 current_volume, key_pressed, 413 current_volume, key_pressed,
544 &processed_data, &processed_data_audio_delay, &new_volume)) { 414 &processed_data, &processed_data_audio_delay, &new_volume)) {
545 DCHECK(processed_data); 415 DCHECK(processed_data);
546 const base::TimeTicks processed_data_capture_time =
547 reference_clock_snapshot - processed_data_audio_delay;
548 for (const auto& track : tracks) {
549 track->Capture(*processed_data,
550 processed_data_capture_time,
551 force_report_nonzero_energy);
552 }
553 416
554 if (new_volume) { 417 level_calculator_.Calculate(*processed_data, force_report_nonzero_energy);
418
419 MediaStreamAudioSource::DeliverDataToTracks(
420 *processed_data, reference_clock_snapshot - processed_data_audio_delay);
421
422 // TODO(xians): This could result in an IPC call being made for each audio
423 // chunk (!). Consider adding throttling logic here.
424 if (new_volume != current_volume) {
555 SetVolume(new_volume); 425 SetVolume(new_volume);
556
557 // Update the |current_volume| to avoid passing the old volume to AGC.
558 current_volume = new_volume; 426 current_volume = new_volume;
559 } 427 }
560 } 428 }
561 } 429 }
562 430
563 void WebRtcAudioCapturer::OnCaptureError(const std::string& message) { 431 void ProcessedLocalAudioSource::OnCaptureError(const std::string& message) {
564 WebRtcLogMessage("WAC::OnCaptureError: " + message); 432 WebRtcLogMessage("ProcessedLocalAudioSource::OnCaptureError: " + message);
565 } 433 }
566 434
567 media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const { 435 media::AudioParameters ProcessedLocalAudioSource::GetInputAudioParameters()
568 base::AutoLock auto_lock(lock_); 436 const {
569 return audio_processor_.get() ? audio_processor_->InputFormat() 437 return audio_processor_.get()
570 : media::AudioParameters(); 438 ? audio_processor_->InputFormat()
571 } 439 : media::AudioParameters();
572
573 bool WebRtcAudioCapturer::GetPairedOutputParameters(
574 int* session_id,
575 int* output_sample_rate,
576 int* output_frames_per_buffer) const {
577 // Don't set output parameters unless all of them are valid.
578 if (device_info_.session_id <= 0 ||
579 !device_info_.device.matched_output.sample_rate ||
580 !device_info_.device.matched_output.frames_per_buffer)
581 return false;
582
583 *session_id = device_info_.session_id;
584 *output_sample_rate = device_info_.device.matched_output.sample_rate;
585 *output_frames_per_buffer =
586 device_info_.device.matched_output.frames_per_buffer;
587
588 return true;
589 }
590
591 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
592 DCHECK(thread_checker_.CalledOnValidThread());
593 #if defined(OS_ANDROID)
594 // TODO(henrika): Tune and adjust buffer size on Android.
595 return (2 * sample_rate / 100);
596 #endif
597
598 // PeerConnection is running at a buffer size of 10ms data. A multiple of
599 // 10ms as the buffer size can give the best performance to PeerConnection.
600 int peer_connection_buffer_size = sample_rate / 100;
601
602 // Use the native hardware buffer size in non peer connection mode when the
603 // platform is using a native buffer size smaller than the PeerConnection
604 // buffer size and audio processing is off.
605 int hardware_buffer_size = device_info_.device.input.frames_per_buffer;
606 if (!peer_connection_mode_ && hardware_buffer_size &&
607 hardware_buffer_size <= peer_connection_buffer_size &&
608 !audio_processor_->has_audio_processing()) {
609 DVLOG(1) << "WebRtcAudioCapturer is using hardware buffer size "
610 << hardware_buffer_size;
611 return hardware_buffer_size;
612 }
613
614 return (sample_rate / 100);
615 }
616
617 void WebRtcAudioCapturer::SetCapturerSource(
618 const scoped_refptr<media::AudioCapturerSource>& source,
619 media::AudioParameters params) {
620 // Create a new audio stream as source which uses the new source.
621 SetCapturerSourceInternal(source,
622 params.channel_layout(),
623 params.sample_rate(),
624 0);
625 } 440 }
626 441
627 } // namespace content 442 } // namespace content
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698