OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #ifndef CONTENT_RENDERER_MEDIA_WEBRTC_AUDIO_CAPTURER_H_ | 5 #ifndef CONTENT_RENDERER_MEDIA_WEBRTC_PROCESSED_LOCAL_AUDIO_SOURCE_H_ |
6 #define CONTENT_RENDERER_MEDIA_WEBRTC_AUDIO_CAPTURER_H_ | 6 #define CONTENT_RENDERER_MEDIA_WEBRTC_PROCESSED_LOCAL_AUDIO_SOURCE_H_ |
7 | 7 |
8 #include <list> | |
9 #include <memory> | |
10 #include <string> | |
11 | |
12 #include "base/callback.h" | |
13 #include "base/files/file.h" | |
14 #include "base/macros.h" | 8 #include "base/macros.h" |
15 #include "base/memory/ref_counted.h" | 9 #include "base/memory/ref_counted.h" |
16 #include "base/synchronization/lock.h" | 10 #include "base/synchronization/lock.h" |
17 #include "base/threading/thread_checker.h" | |
18 #include "base/time/time.h" | |
19 #include "content/common/media/media_stream_options.h" | 11 #include "content/common/media/media_stream_options.h" |
20 #include "content/renderer/media/media_stream_audio_level_calculator.h" | 12 #include "content/renderer/media/media_stream_audio_level_calculator.h" |
21 #include "content/renderer/media/tagged_list.h" | 13 #include "content/renderer/media/media_stream_audio_processor.h" |
22 #include "media/audio/audio_input_device.h" | 14 #include "content/renderer/media/media_stream_audio_source.h" |
23 #include "media/base/audio_capturer_source.h" | 15 #include "media/base/audio_capturer_source.h" |
24 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" | 16 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" |
25 | 17 |
26 namespace media { | 18 namespace media { |
27 class AudioBus; | 19 class AudioBus; |
28 } | 20 } |
29 | 21 |
| 22 namespace webrtc { |
| 23 class AudioSourceInterface; |
| 24 } |
| 25 |
30 namespace content { | 26 namespace content { |
31 | 27 |
32 class MediaStreamAudioProcessor; | 28 class PeerConnectionDependencyFactory; |
33 class MediaStreamAudioSource; | |
34 class WebRtcAudioDeviceImpl; | |
35 class WebRtcLocalAudioRenderer; | |
36 class WebRtcLocalAudioTrack; | |
37 | 29 |
38 // This class manages the capture data flow by getting data from its | 30 // Represents a local source of audio data that is routed through the WebRTC |
39 // |source_|, and passing it to its |tracks_|. | 31 // audio pipeline for post-processing (e.g., for echo cancellation during a |
40 // The threading model for this class is rather complex since it will be | 32 // video conferencing call). Owns a media::AudioCapturerSource and the |
41 // created on the main render thread, captured data is provided on a dedicated | 33 // MediaStreamProcessor that modifies its audio. Modified audio is delivered to |
42 // AudioInputDevice thread, and methods can be called either on the Libjingle | 34 // one or more MediaStreamAudioTracks. |
43 // thread or on the main render thread but also other client threads | 35 class CONTENT_EXPORT ProcessedLocalAudioSource final |
44 // if an alternative AudioCapturerSource has been set. | 36 : NON_EXPORTED_BASE(public MediaStreamAudioSource), |
45 class CONTENT_EXPORT WebRtcAudioCapturer | 37 NON_EXPORTED_BASE(public media::AudioCapturerSource::CaptureCallback) { |
46 : NON_EXPORTED_BASE(public media::AudioCapturerSource::CaptureCallback) { | |
47 public: | 38 public: |
48 // Used to construct the audio capturer. |render_frame_id| specifies the | 39 // |consumer_render_frame_id| references the RenderFrame that will consume the |
49 // RenderFrame consuming audio for capture; -1 is used for tests. | 40 // audio data. Audio parameters and (optionally) a pre-existing audio session |
50 // |device_info| contains all the device information that the capturer is | 41 // ID are derived from |device_info|. |factory| must outlive this instance. |
51 // created for. |constraints| contains the settings for audio processing. | 42 ProcessedLocalAudioSource(int consumer_render_frame_id, |
52 // TODO(xians): Implement the interface for the audio source and move the | 43 const StreamDeviceInfo& device_info, |
53 // |constraints| to ApplyConstraints(). Called on the main render thread. | 44 PeerConnectionDependencyFactory* factory); |
54 static std::unique_ptr<WebRtcAudioCapturer> CreateCapturer( | |
55 int render_frame_id, | |
56 const StreamDeviceInfo& device_info, | |
57 const blink::WebMediaConstraints& constraints, | |
58 WebRtcAudioDeviceImpl* audio_device, | |
59 MediaStreamAudioSource* audio_source); | |
60 | 45 |
61 ~WebRtcAudioCapturer() override; | 46 ~ProcessedLocalAudioSource() final; |
62 | 47 |
63 // Add a audio track to the sinks of the capturer. | 48 // If |source| is an instance of ProcessedLocalAudioSource, return a |
64 // WebRtcAudioDeviceImpl calls this method on the main render thread but | 49 // type-casted pointer to it. Otherwise, return null. |
65 // other clients may call it from other threads. The current implementation | 50 static ProcessedLocalAudioSource* From(MediaStreamAudioSource* source); |
66 // does not support multi-thread calling. | |
67 // The first AddTrack will implicitly trigger the Start() of this object. | |
68 void AddTrack(WebRtcLocalAudioTrack* track); | |
69 | 51 |
70 // Remove a audio track from the sinks of the capturer. | 52 // Non-browser unit tests cannot provide RenderFrame implementations at |
71 // If the track has been added to the capturer, it must call RemoveTrack() | 53 // run-time. This is used to skip the otherwise mandatory check for a valid |
72 // before it goes away. | 54 // render frame ID when the source is started. |
73 // Called on the main render thread or libjingle working thread. | 55 void SetAllowInvalidRenderFrameIdForTesting(bool allowed) { |
74 void RemoveTrack(WebRtcLocalAudioTrack* track); | 56 allow_invalid_render_frame_id_for_testing_ = allowed; |
| 57 } |
75 | 58 |
76 // Called when a stream is connecting to a peer connection. This will set | 59 // Gets/Sets source constraints. Using this is optional, but must be done |
77 // up the native buffer size for the stream in order to optimize the | 60 // before the first call to ConnectToTrack(). |
78 // performance for peer connection. | 61 blink::WebMediaConstraints source_constraints() const { return constraints_; } |
79 void EnablePeerConnectionMode(); | 62 void SetSourceConstraints(const blink::WebMediaConstraints& constraints); |
80 | 63 |
81 // Volume APIs used by WebRtcAudioDeviceImpl. | 64 // The following accessors are not valid until after the source is started |
82 // Called on the AudioInputDevice audio thread. | 65 // (when the first track is connected). |
| 66 webrtc::AudioSourceInterface* rtc_source() const { return rtc_source_.get(); } |
| 67 const scoped_refptr<MediaStreamAudioProcessor>& audio_processor() const { |
| 68 return audio_processor_; |
| 69 } |
| 70 const scoped_refptr<MediaStreamAudioLevelCalculator::Level>& audio_level() |
| 71 const { |
| 72 return level_calculator_.level(); |
| 73 } |
| 74 |
| 75 // Thread-safe volume accessors used by WebRtcAudioDeviceImpl. |
83 void SetVolume(int volume); | 76 void SetVolume(int volume); |
84 int Volume() const; | 77 int Volume() const; |
85 int MaxVolume() const; | 78 int MaxVolume() const; |
86 | 79 |
87 // Audio parameters utilized by the source of the audio capturer. | 80 // Audio parameters utilized by the source of the audio capturer. |
88 // TODO(phoglund): Think over the implications of this accessor and if we can | 81 // TODO(phoglund): Think over the implications of this accessor and if we can |
89 // remove it. | 82 // remove it. |
90 media::AudioParameters GetInputFormat() const; | 83 media::AudioParameters GetInputFormat() const; |
91 | 84 |
92 const StreamDeviceInfo& device_info() const { return device_info_; } | 85 protected: |
93 | 86 // MediaStreamAudioSource implementation. |
94 // Stops recording audio. This method will empty its track lists since | 87 void* GetClassIdentifier() const final; |
95 // stopping the capturer will implicitly invalidate all its tracks. | 88 bool EnsureSourceIsStarted() final; |
96 // This method is exposed to the public because the MediaStreamAudioSource can | 89 void EnsureSourceIsStopped() final; |
97 // call Stop() | |
98 void Stop(); | |
99 | |
100 // Returns the output format. | |
101 // Called on the main render thread. | |
102 media::AudioParameters GetOutputFormat() const; | |
103 | |
104 // Used by clients to inject their own source to the capturer. | |
105 void SetCapturerSource( | |
106 const scoped_refptr<media::AudioCapturerSource>& source, | |
107 media::AudioParameters params); | |
108 | |
109 private: | |
110 class TrackOwner; | |
111 typedef TaggedList<TrackOwner> TrackList; | |
112 | |
113 WebRtcAudioCapturer(int render_frame_id, | |
114 const StreamDeviceInfo& device_info, | |
115 const blink::WebMediaConstraints& constraints, | |
116 WebRtcAudioDeviceImpl* audio_device, | |
117 MediaStreamAudioSource* audio_source); | |
118 | 90 |
119 // AudioCapturerSource::CaptureCallback implementation. | 91 // AudioCapturerSource::CaptureCallback implementation. |
120 // Called on the AudioInputDevice audio thread. | 92 // Called on the AudioCapturerSource audio thread. |
121 void Capture(const media::AudioBus* audio_source, | 93 void Capture(const media::AudioBus* audio_source, |
122 int audio_delay_milliseconds, | 94 int audio_delay_milliseconds, |
123 double volume, | 95 double volume, |
124 bool key_pressed) override; | 96 bool key_pressed) override; |
125 void OnCaptureError(const std::string& message) override; | 97 void OnCaptureError(const std::string& message) override; |
126 | 98 |
127 // Initializes the default audio capturing source using the provided render | 99 private: |
128 // frame id and device information. Return true if success, otherwise false. | 100 // Helper function to get the source buffer size based on whether audio |
129 bool Initialize(); | 101 // processing will take place. |
130 | |
131 // SetCapturerSourceInternal() is called if the client on the source side | |
132 // desires to provide their own captured audio data. Client is responsible | |
133 // for calling Start() on its own source to get the ball rolling. | |
134 // Called on the main render thread. | |
135 // buffer_size is optional. Set to 0 to let it be chosen automatically. | |
136 void SetCapturerSourceInternal( | |
137 const scoped_refptr<media::AudioCapturerSource>& source, | |
138 media::ChannelLayout channel_layout, | |
139 int sample_rate); | |
140 | |
141 // Starts recording audio. | |
142 // Triggered by AddSink() on the main render thread or a Libjingle working | |
143 // thread. It should NOT be called under |lock_|. | |
144 void Start(); | |
145 | |
146 // Helper function to get the buffer size based on |peer_connection_mode_| | |
147 // and sample rate; | |
148 int GetBufferSize(int sample_rate) const; | 102 int GetBufferSize(int sample_rate) const; |
149 | 103 |
150 // Used to DCHECK that we are called on the correct thread. | 104 // The RenderFrame that will consume the audio data. Used when creating |
| 105 // AudioCapturerSources. |
| 106 const int consumer_render_frame_id_; |
| 107 |
| 108 PeerConnectionDependencyFactory* const pc_factory_; |
| 109 |
| 110 // In debug builds, check that all methods that could cause object graph |
| 111 // or data flow changes are being called on the main thread. |
151 base::ThreadChecker thread_checker_; | 112 base::ThreadChecker thread_checker_; |
152 | 113 |
153 // Protects |source_|, |audio_tracks_|, |running_|, |loopback_fifo_|, | |
154 // |params_| and |buffering_|. | |
155 mutable base::Lock lock_; | |
156 | |
157 // A tagged list of audio tracks that the audio data is fed | |
158 // to. Tagged items need to be notified that the audio format has | |
159 // changed. | |
160 TrackList tracks_; | |
161 | |
162 // The audio data source from the browser process. | |
163 scoped_refptr<media::AudioCapturerSource> source_; | |
164 | |
165 // Cached audio constraints for the capturer. | 114 // Cached audio constraints for the capturer. |
166 blink::WebMediaConstraints constraints_; | 115 blink::WebMediaConstraints constraints_; |
167 | 116 |
168 // Audio processor doing processing like FIFO, AGC, AEC and NS. Its output | 117 // Audio processor doing processing like FIFO, AGC, AEC and NS. Its output |
169 // data is in a unit of 10 ms data chunk. | 118 // data is in a unit of 10 ms data chunk. |
170 const scoped_refptr<MediaStreamAudioProcessor> audio_processor_; | 119 scoped_refptr<MediaStreamAudioProcessor> audio_processor_; |
171 | 120 |
172 bool running_; | 121 // The device created by the AudioDeviceFactory in EnsureSourceIsStarted(). |
| 122 scoped_refptr<media::AudioCapturerSource> source_; |
173 | 123 |
174 int render_frame_id_; | 124 // Holder for WebRTC audio pipeline objects. Created in |
| 125 // EnsureSourceIsStarted(). |
| 126 scoped_refptr<webrtc::AudioSourceInterface> rtc_source_; |
175 | 127 |
176 // Cached information of the device used by the capturer. | 128 // Protects data elements from concurrent access when using the volume |
177 const StreamDeviceInfo device_info_; | 129 // methods. |
| 130 mutable base::Lock volume_lock_; |
178 | 131 |
179 // Stores latest microphone volume received in a CaptureData() callback. | 132 // Stores latest microphone volume received in a CaptureData() callback. |
180 // Range is [0, 255]. | 133 // Range is [0, 255]. |
181 int volume_; | 134 int volume_; |
182 | 135 |
183 // Flag which affects the buffer size used by the capturer. | |
184 bool peer_connection_mode_; | |
185 | |
186 // Raw pointer to the WebRtcAudioDeviceImpl, which is valid for the lifetime | |
187 // of RenderThread. | |
188 WebRtcAudioDeviceImpl* audio_device_; | |
189 | |
190 // Raw pointer to the MediaStreamAudioSource object that holds a reference | |
191 // to this WebRtcAudioCapturer. | |
192 // Since |audio_source_| is owned by a blink::WebMediaStreamSource object and | |
193 // blink guarantees that the blink::WebMediaStreamSource outlives any | |
194 // blink::WebMediaStreamTrack connected to the source, |audio_source_| is | |
195 // guaranteed to exist as long as a WebRtcLocalAudioTrack is connected to this | |
196 // WebRtcAudioCapturer. | |
197 MediaStreamAudioSource* const audio_source_; | |
198 | |
199 // Used to calculate the signal level that shows in the UI. | 136 // Used to calculate the signal level that shows in the UI. |
200 MediaStreamAudioLevelCalculator level_calculator_; | 137 MediaStreamAudioLevelCalculator level_calculator_; |
201 | 138 |
202 DISALLOW_COPY_AND_ASSIGN(WebRtcAudioCapturer); | 139 bool allow_invalid_render_frame_id_for_testing_; |
| 140 |
| 141 DISALLOW_COPY_AND_ASSIGN(ProcessedLocalAudioSource); |
203 }; | 142 }; |
204 | 143 |
205 } // namespace content | 144 } // namespace content |
206 | 145 |
207 #endif // CONTENT_RENDERER_MEDIA_WEBRTC_AUDIO_CAPTURER_H_ | 146 #endif // CONTENT_RENDERER_MEDIA_WEBRTC_PROCESSED_LOCAL_AUDIO_SOURCE_H_ |
OLD | NEW |