OLD | NEW |
---|---|
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #ifndef CONTENT_RENDERER_MEDIA_WEBRTC_LOCAL_AUDIO_RENDERER_H_ | 5 #ifndef CONTENT_RENDERER_MEDIA_TRACK_AUDIO_RENDERER_H_ |
6 #define CONTENT_RENDERER_MEDIA_WEBRTC_LOCAL_AUDIO_RENDERER_H_ | 6 #define CONTENT_RENDERER_MEDIA_TRACK_AUDIO_RENDERER_H_ |
7 | 7 |
8 #include <stdint.h> | 8 #include <stdint.h> |
9 | 9 |
10 #include <string> | 10 #include <string> |
11 #include <vector> | 11 #include <vector> |
12 | 12 |
13 #include "base/callback.h" | 13 #include "base/callback.h" |
14 #include "base/macros.h" | 14 #include "base/macros.h" |
15 #include "base/memory/ref_counted.h" | 15 #include "base/memory/ref_counted.h" |
16 #include "base/single_thread_task_runner.h" | 16 #include "base/single_thread_task_runner.h" |
17 #include "base/synchronization/lock.h" | 17 #include "base/synchronization/lock.h" |
18 #include "base/threading/thread_checker.h" | 18 #include "base/threading/thread_checker.h" |
19 #include "content/common/content_export.h" | 19 #include "content/common/content_export.h" |
20 #include "content/public/renderer/media_stream_audio_renderer.h" | 20 #include "content/public/renderer/media_stream_audio_renderer.h" |
21 #include "content/public/renderer/media_stream_audio_sink.h" | 21 #include "content/public/renderer/media_stream_audio_sink.h" |
22 #include "content/renderer/media/webrtc_audio_device_impl.h" | 22 #include "media/base/audio_renderer_sink.h" |
23 #include "content/renderer/media/webrtc_local_audio_track.h" | |
24 #include "media/base/output_device.h" | 23 #include "media/base/output_device.h" |
25 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" | 24 #include "third_party/WebKit/public/platform/WebMediaStreamTrack.h" |
26 | 25 |
27 namespace media { | 26 namespace media { |
28 class AudioBus; | 27 class AudioBus; |
29 class AudioShifter; | 28 class AudioShifter; |
30 class AudioOutputDevice; | 29 class AudioOutputDevice; |
31 class AudioParameters; | 30 class AudioParameters; |
32 } | 31 } |
33 | 32 |
34 namespace content { | 33 namespace content { |
35 | 34 |
36 class WebRtcAudioCapturer; | 35 // TrackAudioRenderer is a MediaStreamAudioRenderer for plumbing audio data |
37 | 36 // generated from either local or remote MediaStreamAudioTracks to an audio |
o1ka
2016/02/10 10:24:13
It would be nice to add that remote tracks are "no
miu
2016/02/10 21:43:12
Done.
| |
38 // WebRtcLocalAudioRenderer is a MediaStreamAudioRenderer designed for rendering | 37 // output device, reconciling differences in the rates of production and |
39 // local audio media stream tracks, | 38 // consumption of the audio data. |
40 // http://dev.w3.org/2011/webrtc/editor/getusermedia.html#mediastreamtrack | 39 // |
41 // It also implements media::AudioRendererSink::RenderCallback to render audio | 40 // This class uses AudioDeviceFactory to create media::AudioOutputDevices and |
42 // data provided from a WebRtcLocalAudioTrack source. | 41 // owns/manages their lifecycles. Output devices are automatically re-created |
43 // When the audio layer in the browser process asks for data to render, this | 42 // in response to audio format changes, or use of the SwitchOutputDevice() API |
44 // class provides the data by implementing the MediaStreamAudioSink | 43 // by client code. |
45 // interface, i.e., we are a sink seen from the WebRtcAudioCapturer perspective. | 44 // |
46 // TODO(henrika): improve by using similar principles as in | 45 // Audio data is feed-in from the source via calls to OnData(). The |
47 // MediaStreamVideoRendererSink which register itself to the video track when | 46 // internally-owned media::AudioOutputDevice calls Render() to pull-out that |
48 // the provider is started and deregisters itself when it is stopped. Tracking | 47 // audio data. However, because of clock differences and other environmental |
49 // this at http://crbug.com/164813. | 48 // factors, the audio will inevitably feed-in at a rate different from the rate |
50 class CONTENT_EXPORT WebRtcLocalAudioRenderer | 49 // it is being rendered-out. media::AudioShifter is used to buffer, stretch |
50 // and skip audio to maintain time synchronization between the producer and | |
51 // consumer. | |
52 class CONTENT_EXPORT TrackAudioRenderer | |
51 : NON_EXPORTED_BASE(public MediaStreamAudioRenderer), | 53 : NON_EXPORTED_BASE(public MediaStreamAudioRenderer), |
52 NON_EXPORTED_BASE(public MediaStreamAudioSink), | 54 NON_EXPORTED_BASE(public MediaStreamAudioSink), |
53 NON_EXPORTED_BASE(public media::AudioRendererSink::RenderCallback), | 55 NON_EXPORTED_BASE(public media::AudioRendererSink::RenderCallback), |
54 NON_EXPORTED_BASE(public media::OutputDevice) { | 56 NON_EXPORTED_BASE(public media::OutputDevice) { |
55 public: | 57 public: |
56 // Creates a local renderer and registers a capturing |source| object. | 58 // Creates a renderer for the given |audio_track|. |playout_render_frame_id| |
57 // The |source| is owned by the WebRtcAudioDeviceImpl. | 59 // refers to the RenderFrame that owns this instance (e.g., it contains the |
60 // DOM widget representing the player). |session_id| and |device_id| are | |
61 // optional, and are used to direct audio output to a pre-selected device; | |
62 // otherwise, audio is output to the default device for the system. | |
63 // | |
58 // Called on the main thread. | 64 // Called on the main thread. |
59 WebRtcLocalAudioRenderer(const blink::WebMediaStreamTrack& audio_track, | 65 TrackAudioRenderer(const blink::WebMediaStreamTrack& audio_track, |
60 int source_render_frame_id, | 66 int playout_render_frame_id, |
61 int session_id, | 67 int session_id, |
62 const std::string& device_id, | 68 const std::string& device_id, |
63 const url::Origin& security_origin); | 69 const url::Origin& security_origin); |
64 | 70 |
65 // MediaStreamAudioRenderer implementation. | 71 // MediaStreamAudioRenderer implementation. |
66 // Called on the main thread. | 72 // Called on the main thread. |
67 void Start() override; | 73 void Start() override; |
68 void Stop() override; | 74 void Stop() override; |
69 void Play() override; | 75 void Play() override; |
70 void Pause() override; | 76 void Pause() override; |
71 void SetVolume(float volume) override; | 77 void SetVolume(float volume) override; |
72 media::OutputDevice* GetOutputDevice() override; | 78 media::OutputDevice* GetOutputDevice() override; |
73 base::TimeDelta GetCurrentRenderTime() const override; | 79 base::TimeDelta GetCurrentRenderTime() const override; |
74 bool IsLocalRenderer() const override; | 80 bool IsLocalRenderer() const override; |
75 | 81 |
76 // media::OutputDevice implementation | 82 // media::OutputDevice implementation |
77 void SwitchOutputDevice(const std::string& device_id, | 83 void SwitchOutputDevice(const std::string& device_id, |
78 const url::Origin& security_origin, | 84 const url::Origin& security_origin, |
79 const media::SwitchOutputDeviceCB& callback) override; | 85 const media::SwitchOutputDeviceCB& callback) override; |
80 media::AudioParameters GetOutputParameters() override; | 86 media::AudioParameters GetOutputParameters() override; |
81 media::OutputDeviceStatus GetDeviceStatus() override; | 87 media::OutputDeviceStatus GetDeviceStatus() override; |
82 | 88 |
83 const base::TimeDelta& total_render_time() const { | |
84 return total_render_time_; | |
85 } | |
86 | |
87 protected: | 89 protected: |
88 ~WebRtcLocalAudioRenderer() override; | 90 ~TrackAudioRenderer() override; |
89 | 91 |
90 private: | 92 private: |
91 // MediaStreamAudioSink implementation. | 93 // MediaStreamAudioSink implementation. |
92 | 94 |
93 // Called on the AudioInputDevice worker thread. | 95 // Called on the AudioInputDevice worker thread. |
94 void OnData(const media::AudioBus& audio_bus, | 96 void OnData(const media::AudioBus& audio_bus, |
95 base::TimeTicks estimated_capture_time) override; | 97 base::TimeTicks reference_time) override; |
96 | 98 |
97 // Called on the AudioInputDevice worker thread. | 99 // Called on the AudioInputDevice worker thread. |
98 void OnSetFormat(const media::AudioParameters& params) override; | 100 void OnSetFormat(const media::AudioParameters& params) override; |
99 | 101 |
100 // media::AudioRendererSink::RenderCallback implementation. | 102 // media::AudioRendererSink::RenderCallback implementation. |
101 // Render() is called on the AudioOutputDevice thread and OnRenderError() | 103 // Render() is called on the AudioOutputDevice thread and OnRenderError() |
102 // on the IO thread. | 104 // on the IO thread. |
103 int Render(media::AudioBus* audio_bus, | 105 int Render(media::AudioBus* audio_bus, |
104 uint32_t audio_delay_milliseconds, | 106 uint32_t audio_delay_milliseconds, |
105 uint32_t frames_skipped) override; | 107 uint32_t frames_skipped) override; |
106 void OnRenderError() override; | 108 void OnRenderError() override; |
107 | 109 |
108 // Initializes and starts the |sink_| if | 110 // Initializes and starts the |sink_| if |
109 // we have received valid |source_params_| && | 111 // we have received valid |source_params_| && |
110 // |playing_| has been set to true && | 112 // |playing_| has been set to true. |
111 // |volume_| is not zero. | |
112 void MaybeStartSink(); | 113 void MaybeStartSink(); |
113 | 114 |
114 // Sets new |source_params_| and then re-initializes and restarts |sink_|. | 115 // Sets new |source_params_| and then re-initializes and restarts |sink_|. |
115 void ReconfigureSink(const media::AudioParameters& params); | 116 void ReconfigureSink(const media::AudioParameters& params); |
116 | 117 |
117 // The audio track which provides data to render. Given that this class | 118 // Creates a new AudioShifter, destroying the old one (if any). This is |
118 // implements local loopback, the audio track is getting data from a capture | 119 // called any time playback is started/stopped, or the sink changes. |
119 // instance like a selected microphone and forwards the recorded data to its | 120 void CreateAudioShifter(); |
120 // sinks. The recorded data is stored in a FIFO and consumed | 121 |
121 // by this class when the sink asks for new data. | 122 // Called when either the source or sink has changed somehow, or audio has |
123 // been paused. Drops the AudioShifter and updates | |
124 // |prior_elapsed_render_time_|. May be called from either the main thread or | |
125 // the audio thread. Assumption: |thread_lock_| is already acquired. | |
126 void HaltAudioFlowWhileLockHeld(); | |
127 | |
128 // The audio track which provides access to the source data to render. | |
129 // | |
122 // This class is calling MediaStreamAudioSink::AddToAudioTrack() and | 130 // This class is calling MediaStreamAudioSink::AddToAudioTrack() and |
123 // MediaStreamAudioSink::RemoveFromAudioTrack() to connect and disconnect | 131 // MediaStreamAudioSink::RemoveFromAudioTrack() to connect and disconnect |
124 // with the audio track. | 132 // with the audio track. |
125 blink::WebMediaStreamTrack audio_track_; | 133 blink::WebMediaStreamTrack audio_track_; |
126 | 134 |
127 // The render view and frame in which the audio is rendered into |sink_|. | 135 // The render view and frame in which the audio is rendered into |sink_|. |
128 const int source_render_frame_id_; | 136 const int playout_render_frame_id_; |
129 const int session_id_; | 137 const int session_id_; |
130 | 138 |
131 // MessageLoop associated with the single thread that performs all control | 139 // MessageLoop associated with the single thread that performs all control |
132 // tasks. Set to the MessageLoop that invoked the ctor. | 140 // tasks. Set to the MessageLoop that invoked the ctor. |
133 const scoped_refptr<base::SingleThreadTaskRunner> task_runner_; | 141 const scoped_refptr<base::SingleThreadTaskRunner> task_runner_; |
134 | 142 |
135 // The sink (destination) for rendered audio. | 143 // The sink (destination) for rendered audio. |
136 scoped_refptr<media::AudioOutputDevice> sink_; | 144 scoped_refptr<media::AudioOutputDevice> sink_; |
137 | 145 |
138 // This does all the synchronization/resampling/smoothing. | 146 // This does all the synchronization/resampling/smoothing. |
139 scoped_ptr<media::AudioShifter> audio_shifter_; | 147 scoped_ptr<media::AudioShifter> audio_shifter_; |
140 | 148 |
141 // Stores last time a render callback was received. The time difference | 149 // These track the time duration of all the audio rendered so far by this |
142 // between a new time stamp and this value can be used to derive the | 150 // instance. |prior_elapsed_render_time_| tracks the time duration of all |
143 // total render time. | 151 // audio rendered before the last format change. |num_samples_rendered_| |
144 base::TimeTicks last_render_time_; | 152 // tracks the number of audio samples rendered since the last format change. |
153 base::TimeDelta prior_elapsed_render_time_; | |
154 int64_t num_samples_rendered_; | |
145 | 155 |
146 // Keeps track of total time audio has been rendered. | 156 // The audio parameters of the track's source. |
147 base::TimeDelta total_render_time_; | |
148 | |
149 // The audio parameters of the capture source. | |
150 // Must only be touched on the main thread. | 157 // Must only be touched on the main thread. |
151 media::AudioParameters source_params_; | 158 media::AudioParameters source_params_; |
152 | 159 |
153 // The audio parameters used by the sink. | |
154 // Must only be touched on the main thread. | |
155 media::AudioParameters sink_params_; | |
156 | |
157 // Set when playing, cleared when paused. | 160 // Set when playing, cleared when paused. |
158 bool playing_; | 161 bool playing_; |
159 | 162 |
160 // Protects |audio_shifter_|, |playing_|, |last_render_time_|, | 163 // Protects |audio_shifter_|, |prior_elapsed_render_time_|, and |
161 // |total_render_time_| and |volume_|. | 164 // |num_samples_rendered_|. |
162 mutable base::Lock thread_lock_; | 165 mutable base::Lock thread_lock_; |
163 | 166 |
164 // The preferred device id of the output device or empty for the default | 167 // The preferred device id of the output device or empty for the default |
165 // output device. | 168 // output device. |
166 std::string output_device_id_; | 169 std::string output_device_id_; |
167 url::Origin security_origin_; | 170 url::Origin security_origin_; |
168 | 171 |
169 // Cache value for the volume. | 172 // Cache value for the volume. Whenever |sink_| is re-created, its volume |
173 // should be set to this. | |
170 float volume_; | 174 float volume_; |
171 | 175 |
172 // Flag to indicate whether |sink_| has been started yet. | 176 // Flag to indicate whether |sink_| has been started yet. |
173 bool sink_started_; | 177 bool sink_started_; |
174 | 178 |
175 // Used to DCHECK that some methods are called on the capture audio thread. | 179 // Used to DCHECK that some methods are called on the audio thread. |
176 base::ThreadChecker capture_thread_checker_; | 180 base::ThreadChecker audio_thread_checker_; |
177 | 181 |
178 DISALLOW_COPY_AND_ASSIGN(WebRtcLocalAudioRenderer); | 182 DISALLOW_COPY_AND_ASSIGN(TrackAudioRenderer); |
179 }; | 183 }; |
180 | 184 |
181 } // namespace content | 185 } // namespace content |
182 | 186 |
183 #endif // CONTENT_RENDERER_MEDIA_WEBRTC_LOCAL_AUDIO_RENDERER_H_ | 187 #endif // CONTENT_RENDERER_MEDIA_TRACK_AUDIO_RENDERER_H_ |
OLD | NEW |