OLD | NEW |
---|---|
1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "remoting/protocol/audio_pump.h" | 5 #include "remoting/protocol/audio_pump.h" |
6 | 6 |
7 #include <memory> | |
7 #include <utility> | 8 #include <utility> |
8 | 9 |
9 #include "base/bind.h" | 10 #include "base/bind.h" |
10 #include "base/location.h" | 11 #include "base/location.h" |
11 #include "base/logging.h" | 12 #include "base/logging.h" |
12 #include "base/macros.h" | 13 #include "base/macros.h" |
14 #include "base/memory/ptr_util.h" | |
13 #include "base/single_thread_task_runner.h" | 15 #include "base/single_thread_task_runner.h" |
14 #include "base/threading/thread_task_runner_handle.h" | 16 #include "base/threading/thread_task_runner_handle.h" |
17 #include "media/base/audio_bus.h" | |
18 #include "media/base/audio_sample_types.h" | |
19 #include "media/base/channel_layout.h" | |
20 #include "media/base/channel_mixer.h" | |
15 #include "remoting/codec/audio_encoder.h" | 21 #include "remoting/codec/audio_encoder.h" |
16 #include "remoting/proto/audio.pb.h" | 22 #include "remoting/proto/audio.pb.h" |
17 #include "remoting/protocol/audio_source.h" | 23 #include "remoting/protocol/audio_source.h" |
18 #include "remoting/protocol/audio_stub.h" | 24 #include "remoting/protocol/audio_stub.h" |
19 | 25 |
26 namespace { | |
27 | |
28 int CalculateFrameCount(const remoting::AudioPacket& packet) { | |
29 return packet.data(0).size() / packet.channels() / packet.bytes_per_sample(); | |
30 } | |
31 | |
32 std::unique_ptr<media::AudioBus> AudioPacketToAudioBus( | |
33 const remoting::AudioPacket& packet) { | |
34 const int frame_count = CalculateFrameCount(packet); | |
35 if (frame_count < 1) { | |
Sergey Ulanov
2017/06/02 18:34:36
packet.data(0).is_empty()
Sergey Ulanov
2017/06/02 18:34:36
Why would we get an empty packet here? Can it be r
Hzj_jie
2017/06/02 22:11:25
(frame_count == 0) != (packet.data(0).empty())
Con
Hzj_jie
2017/06/02 22:11:25
It's not an empty packet, but a defeated packet.
Sergey Ulanov
2017/06/05 18:03:19
Then this should be a DCHECK. We own AudioCapturer
Hzj_jie
2017/06/06 03:21:42
I still doubt this decision, especially the DCHECK
Sergey Ulanov
2017/06/07 06:24:59
Do you have an examples when that would make a dif
Hzj_jie
2017/06/07 20:27:18
IMO, changing the following or similar pattern:
==
Sergey Ulanov
2017/06/07 21:16:29
See https://chromium.googlesource.com/chromium/src
| |
36 return nullptr; | |
37 } | |
38 if (static_cast<int>(packet.data(0).size()) >= | |
39 media::AudioBus::CalculateMemorySize(packet.channels(), frame_count)) { | |
40 // TODO(zijiehe): AudioBus::BuildChannelData() and WrapMemory() should | |
41 // receive "const float*" instead of "float*". We can remove the const_cast | |
42 // afterwards. | |
43 return media::AudioBus::WrapMemory( | |
Sergey Ulanov
2017/06/02 18:34:36
Does this work correctly
remoting::AudioPacket alw
Hzj_jie
2017/06/02 22:11:25
I just realize this condition won't meet anyway. I
| |
44 packet.channels(), | |
45 frame_count, | |
46 reinterpret_cast<int16_t*>(const_cast<char*>(packet.data(0).data()))); | |
Sergey Ulanov
2017/06/02 18:34:37
You can use string_as_array() from base/stl_util.h
Hzj_jie
2017/06/02 22:11:25
Done.
| |
47 } | |
48 | |
49 std::unique_ptr<media::AudioBus> result = | |
50 media::AudioBus::Create(packet.channels(), frame_count); | |
51 result->FromInterleaved<media::SignedInt16SampleTypeTraits>( | |
52 reinterpret_cast<const int16_t*>(packet.data(0).data()), frame_count); | |
53 return result; | |
54 } | |
55 | |
56 std::unique_ptr<remoting::AudioPacket> AudioBusToAudioPacket( | |
57 const media::AudioBus& packet) { | |
58 std::unique_ptr<remoting::AudioPacket> result = | |
59 base::MakeUnique<remoting::AudioPacket>(); | |
60 result->add_data()->resize( | |
61 media::AudioBus::CalculateMemorySize(packet.channels(), packet.frames()) / | |
62 sizeof(float) * sizeof(int16_t)); | |
Sergey Ulanov
2017/06/02 18:34:36
I don't think we want to use AudioBus::CalculateMe
Hzj_jie
2017/06/02 22:11:25
Done.
| |
63 packet.ToInterleaved<media::SignedInt16SampleTypeTraits>( | |
64 packet.frames(), | |
65 reinterpret_cast<int16_t*>(&(result->mutable_data(0)->at(0)))); | |
66 return result; | |
67 } | |
68 | |
69 media::ChannelLayout RetrieveLayout(const remoting::AudioPacket& packet) { | |
70 // This switch should match AudioPacket::Channels enum in audio.proto. | |
71 switch (packet.channels()) { | |
72 case remoting::AudioPacket::CHANNELS_INVALID: | |
73 return media::CHANNEL_LAYOUT_UNSUPPORTED; | |
74 case remoting::AudioPacket::CHANNELS_MONO: | |
75 return media::CHANNEL_LAYOUT_MONO; | |
76 case remoting::AudioPacket::CHANNELS_STEREO: | |
77 return media::CHANNEL_LAYOUT_STEREO; | |
78 case remoting::AudioPacket::CHANNELS_SURROUND: | |
79 return media::CHANNEL_LAYOUT_SURROUND; | |
80 case remoting::AudioPacket::CHANNELS_QUAD: | |
81 return media::CHANNEL_LAYOUT_QUAD; | |
Sergey Ulanov
2017/06/02 18:34:36
I don't think that downmixing all 4-channel layout
Hzj_jie
2017/06/02 22:11:25
Done.
| |
82 case remoting::AudioPacket::CHANNELS_4_1: | |
83 return media::CHANNEL_LAYOUT_4_1; | |
84 case remoting::AudioPacket::CHANNELS_5_1: | |
85 return media::CHANNEL_LAYOUT_5_1; | |
86 case remoting::AudioPacket::CHANNELS_6_1: | |
87 return media::CHANNEL_LAYOUT_6_1; | |
88 case remoting::AudioPacket::CHANNELS_7_1: | |
89 return media::CHANNEL_LAYOUT_7_1; | |
90 } | |
91 NOTREACHED() << "Invalid AudioPacket::Channels"; | |
92 return media::CHANNEL_LAYOUT_UNSUPPORTED; | |
93 } | |
94 | |
95 } // namespace | |
96 | |
20 namespace remoting { | 97 namespace remoting { |
21 namespace protocol { | 98 namespace protocol { |
22 | 99 |
23 // Limit the data stored in the pending send buffers to 250ms. | 100 // Limit the data stored in the pending send buffers to 250ms. |
24 const int kMaxBufferedIntervalMs = 250; | 101 const int kMaxBufferedIntervalMs = 250; |
25 | 102 |
26 class AudioPump::Core { | 103 class AudioPump::Core { |
27 public: | 104 public: |
28 Core(base::WeakPtr<AudioPump> pump, | 105 Core(base::WeakPtr<AudioPump> pump, |
29 std::unique_ptr<AudioSource> audio_source, | 106 std::unique_ptr<AudioSource> audio_source, |
30 std::unique_ptr<AudioEncoder> audio_encoder); | 107 std::unique_ptr<AudioEncoder> audio_encoder); |
31 ~Core(); | 108 ~Core(); |
32 | 109 |
33 void Start(); | 110 void Start(); |
34 void Pause(bool pause); | 111 void Pause(bool pause); |
35 | 112 |
36 void OnPacketSent(int size); | 113 void OnPacketSent(int size); |
37 | 114 |
38 private: | 115 private: |
116 std::unique_ptr<AudioPacket> Downmix(std::unique_ptr<AudioPacket> packet); | |
117 | |
39 void EncodeAudioPacket(std::unique_ptr<AudioPacket> packet); | 118 void EncodeAudioPacket(std::unique_ptr<AudioPacket> packet); |
40 | 119 |
41 base::ThreadChecker thread_checker_; | 120 base::ThreadChecker thread_checker_; |
42 | 121 |
43 base::WeakPtr<AudioPump> pump_; | 122 base::WeakPtr<AudioPump> pump_; |
44 | 123 |
45 scoped_refptr<base::SingleThreadTaskRunner> pump_task_runner_; | 124 scoped_refptr<base::SingleThreadTaskRunner> pump_task_runner_; |
46 | 125 |
47 std::unique_ptr<AudioSource> audio_source_; | 126 std::unique_ptr<AudioSource> audio_source_; |
48 std::unique_ptr<AudioEncoder> audio_encoder_; | 127 std::unique_ptr<AudioEncoder> audio_encoder_; |
49 | 128 |
50 bool enabled_; | 129 bool enabled_; |
51 | 130 |
52 // Number of bytes in the queue that have been encoded but haven't been sent | 131 // Number of bytes in the queue that have been encoded but haven't been sent |
53 // yet. | 132 // yet. |
54 int bytes_pending_; | 133 int bytes_pending_; |
55 | 134 |
135 std::unique_ptr<media::ChannelMixer> mixer_; | |
136 media::ChannelLayout last_input_layout_ = media::CHANNEL_LAYOUT_NONE; | |
137 | |
56 DISALLOW_COPY_AND_ASSIGN(Core); | 138 DISALLOW_COPY_AND_ASSIGN(Core); |
57 }; | 139 }; |
58 | 140 |
59 AudioPump::Core::Core(base::WeakPtr<AudioPump> pump, | 141 AudioPump::Core::Core(base::WeakPtr<AudioPump> pump, |
60 std::unique_ptr<AudioSource> audio_source, | 142 std::unique_ptr<AudioSource> audio_source, |
61 std::unique_ptr<AudioEncoder> audio_encoder) | 143 std::unique_ptr<AudioEncoder> audio_encoder) |
62 : pump_(pump), | 144 : pump_(pump), |
63 pump_task_runner_(base::ThreadTaskRunnerHandle::Get()), | 145 pump_task_runner_(base::ThreadTaskRunnerHandle::Get()), |
64 audio_source_(std::move(audio_source)), | 146 audio_source_(std::move(audio_source)), |
65 audio_encoder_(std::move(audio_encoder)), | 147 audio_encoder_(std::move(audio_encoder)), |
(...skipping 25 matching lines...) Expand all Loading... | |
91 bytes_pending_ -= size; | 173 bytes_pending_ -= size; |
92 DCHECK_GE(bytes_pending_, 0); | 174 DCHECK_GE(bytes_pending_, 0); |
93 } | 175 } |
94 | 176 |
95 void AudioPump::Core::EncodeAudioPacket(std::unique_ptr<AudioPacket> packet) { | 177 void AudioPump::Core::EncodeAudioPacket(std::unique_ptr<AudioPacket> packet) { |
96 DCHECK(thread_checker_.CalledOnValidThread()); | 178 DCHECK(thread_checker_.CalledOnValidThread()); |
97 DCHECK(packet); | 179 DCHECK(packet); |
98 | 180 |
99 int max_buffered_bytes = | 181 int max_buffered_bytes = |
100 audio_encoder_->GetBitrate() * kMaxBufferedIntervalMs / 1000 / 8; | 182 audio_encoder_->GetBitrate() * kMaxBufferedIntervalMs / 1000 / 8; |
101 if (!enabled_ || bytes_pending_ > max_buffered_bytes) | 183 if (!enabled_ || bytes_pending_ > max_buffered_bytes) { |
102 return; | 184 return; |
185 } | |
186 | |
187 if (packet->channels() > AudioPacket::CHANNELS_STEREO) { | |
188 packet = Downmix(std::move(packet)); | |
189 } | |
103 | 190 |
104 std::unique_ptr<AudioPacket> encoded_packet = | 191 std::unique_ptr<AudioPacket> encoded_packet = |
105 audio_encoder_->Encode(std::move(packet)); | 192 audio_encoder_->Encode(std::move(packet)); |
106 | 193 |
107 // The audio encoder returns a null audio packet if there's no audio to send. | 194 // The audio encoder returns a null audio packet if there's no audio to send. |
108 if (!encoded_packet) | 195 if (!encoded_packet) |
109 return; | 196 return; |
110 | 197 |
111 int packet_size = encoded_packet->ByteSize(); | 198 int packet_size = encoded_packet->ByteSize(); |
112 bytes_pending_ += packet_size; | 199 bytes_pending_ += packet_size; |
113 | 200 |
114 pump_task_runner_->PostTask( | 201 pump_task_runner_->PostTask( |
115 FROM_HERE, base::Bind(&AudioPump::SendAudioPacket, pump_, | 202 FROM_HERE, base::Bind(&AudioPump::SendAudioPacket, pump_, |
116 base::Passed(&encoded_packet), packet_size)); | 203 base::Passed(&encoded_packet), packet_size)); |
117 } | 204 } |
118 | 205 |
206 std::unique_ptr<AudioPacket> AudioPump::Core::Downmix( | |
207 std::unique_ptr<AudioPacket> packet) { | |
208 DCHECK(thread_checker_.CalledOnValidThread()); | |
209 DCHECK(packet); | |
210 if (packet->data_size() != 1 || | |
211 packet->data(0).empty() || | |
212 packet->bytes_per_sample() != AudioPacket::BYTES_PER_SAMPLE_2) { | |
Sergey Ulanov
2017/06/02 18:34:36
I think all these checks can be DCHECKs
Hzj_jie
2017/06/02 22:11:25
I have removed packet->data(0).empty() check, it w
| |
213 return packet; | |
214 } | |
215 const media::ChannelLayout input_layout = RetrieveLayout(*packet); | |
216 if (input_layout == media::CHANNEL_LAYOUT_UNSUPPORTED || | |
Sergey Ulanov
2017/06/02 18:34:37
I think in case the layout is unsupported it doesn
Hzj_jie
2017/06/02 22:11:25
Done. But IMO, unsupported by media::ChannelMixer
| |
217 input_layout == media::CHANNEL_LAYOUT_MONO || | |
218 input_layout == media::CHANNEL_LAYOUT_STEREO) { | |
219 return packet; | |
220 } | |
221 | |
222 if (!mixer_ || last_input_layout_ != input_layout) { | |
Sergey Ulanov
2017/06/02 18:34:36
It's better to move this code above mono/stereo ch
Hzj_jie
2017/06/02 22:11:25
If the input_layout is mono or stereo, we won't ne
| |
223 last_input_layout_ = input_layout; | |
224 mixer_ = base::MakeUnique<media::ChannelMixer>( | |
225 input_layout, media::CHANNEL_LAYOUT_STEREO); | |
226 } | |
227 | |
228 std::unique_ptr<media::AudioBus> input = AudioPacketToAudioBus(*packet); | |
229 if (!input) { | |
230 return packet; | |
Sergey Ulanov
2017/06/02 18:34:36
Why would AudioPacketToAudioBus() fail? If it does
Hzj_jie
2017/06/02 22:11:25
A corrupted data without a frame.
| |
231 } | |
232 std::unique_ptr<media::AudioBus> output = | |
233 media::AudioBus::Create(AudioPacket::CHANNELS_STEREO, input->frames()); | |
234 mixer_->Transform(input.get(), output.get()); | |
235 | |
236 std::unique_ptr<AudioPacket> result = AudioBusToAudioPacket(*output); | |
237 result->set_encoding(AudioPacket::ENCODING_RAW); | |
238 result->set_sampling_rate(packet->sampling_rate()); | |
239 result->set_bytes_per_sample(packet->bytes_per_sample()); | |
240 result->set_channels(AudioPacket::CHANNELS_STEREO); | |
Sergey Ulanov
2017/06/02 18:34:37
I think it would be best to set all these properti
Hzj_jie
2017/06/02 22:11:24
There is no sampling rate in AudioBus, so I leave
| |
241 return result; | |
242 } | |
243 | |
119 AudioPump::AudioPump( | 244 AudioPump::AudioPump( |
120 scoped_refptr<base::SingleThreadTaskRunner> audio_task_runner, | 245 scoped_refptr<base::SingleThreadTaskRunner> audio_task_runner, |
121 std::unique_ptr<AudioSource> audio_source, | 246 std::unique_ptr<AudioSource> audio_source, |
122 std::unique_ptr<AudioEncoder> audio_encoder, | 247 std::unique_ptr<AudioEncoder> audio_encoder, |
123 AudioStub* audio_stub) | 248 AudioStub* audio_stub) |
124 : audio_task_runner_(audio_task_runner), | 249 : audio_task_runner_(audio_task_runner), |
125 audio_stub_(audio_stub), | 250 audio_stub_(audio_stub), |
126 weak_factory_(this) { | 251 weak_factory_(this) { |
127 DCHECK(audio_stub_); | 252 DCHECK(audio_stub_); |
128 | 253 |
(...skipping 28 matching lines...) Expand all Loading... | |
157 } | 282 } |
158 | 283 |
159 void AudioPump::OnPacketSent(int size) { | 284 void AudioPump::OnPacketSent(int size) { |
160 audio_task_runner_->PostTask( | 285 audio_task_runner_->PostTask( |
161 FROM_HERE, | 286 FROM_HERE, |
162 base::Bind(&Core::OnPacketSent, base::Unretained(core_.get()), size)); | 287 base::Bind(&Core::OnPacketSent, base::Unretained(core_.get()), size)); |
163 } | 288 } |
164 | 289 |
165 } // namespace protocol | 290 } // namespace protocol |
166 } // namespace remoting | 291 } // namespace remoting |
OLD | NEW |