OLD | NEW |
---|---|
1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/cast/sender/vp8_encoder.h" | 5 #include "media/cast/sender/vp8_encoder.h" |
6 | 6 |
7 #include <vector> | 7 #include <vector> |
8 | 8 |
9 #include "base/logging.h" | 9 #include "base/logging.h" |
10 #include "media/base/video_frame.h" | 10 #include "media/base/video_frame.h" |
11 #include "media/cast/cast_defines.h" | 11 #include "media/cast/cast_defines.h" |
12 #include "media/cast/net/cast_transport_config.h" | 12 #include "media/cast/net/cast_transport_config.h" |
13 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" | 13 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" |
14 | 14 |
15 namespace media { | 15 namespace media { |
16 namespace cast { | 16 namespace cast { |
17 | 17 |
18 static const uint32 kMinIntra = 300; | 18 static const uint32 kMinIntra = 300; |
19 | 19 |
20 static int ComputeMaxNumOfRepeatedBuffes(int max_unacked_frames) { | |
21 if (max_unacked_frames > kNumberOfVp8VideoBuffers) | |
22 return (max_unacked_frames - 1) / kNumberOfVp8VideoBuffers; | |
23 | |
24 return 0; | |
25 } | |
26 | |
27 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config, | 20 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config, |
28 int max_unacked_frames) | 21 int max_unacked_frames) |
29 : cast_config_(video_config), | 22 : cast_config_(video_config), |
30 use_multiple_video_buffers_( | 23 use_multiple_video_buffers_( |
31 cast_config_.max_number_of_video_buffers_used == | 24 cast_config_.max_number_of_video_buffers_used == |
32 kNumberOfVp8VideoBuffers), | 25 kNumberOfVp8VideoBuffers), |
33 max_number_of_repeated_buffers_in_a_row_( | |
34 ComputeMaxNumOfRepeatedBuffes(max_unacked_frames)), | |
35 key_frame_requested_(true), | 26 key_frame_requested_(true), |
36 first_frame_received_(false), | 27 first_frame_received_(false), |
37 last_encoded_frame_id_(kStartFrameId), | 28 last_encoded_frame_id_(kStartFrameId), |
38 number_of_repeated_buffers_(0) { | 29 last_acked_frame_id_(kStartFrameId), |
39 // TODO(pwestin): we need to figure out how to synchronize the acking with the | 30 frame_id_to_reference_(kStartFrameId - 1), |
40 // internal state of the encoder, ideally the encoder will tell if we can | 31 undroppable_frames_(0) { |
41 // send another frame. | |
42 DCHECK(!use_multiple_video_buffers_ || | |
43 max_number_of_repeated_buffers_in_a_row_ == 0) | |
44 << "Invalid config"; | |
45 | |
46 // VP8 have 3 buffers available for prediction, with | 32 // VP8 have 3 buffers available for prediction, with |
47 // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency | 33 // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency |
48 // however in this mode we can not skip frames in the receiver to catch up | 34 // however in this mode we can not skip frames in the receiver to catch up |
49 // after a temporary network outage; with max_number_of_video_buffers_used | 35 // after a temporary network outage; with max_number_of_video_buffers_used |
50 // set to 3 we allow 2 frames to be skipped by the receiver without error | 36 // set to 3 we allow 2 frames to be skipped by the receiver without error |
51 // propagation. | 37 // propagation. |
52 DCHECK(cast_config_.max_number_of_video_buffers_used == 1 || | 38 DCHECK(cast_config_.max_number_of_video_buffers_used == 1 || |
53 cast_config_.max_number_of_video_buffers_used == | 39 cast_config_.max_number_of_video_buffers_used == |
54 kNumberOfVp8VideoBuffers) | 40 kNumberOfVp8VideoBuffers) |
55 << "Invalid argument"; | 41 << "Invalid argument"; |
(...skipping 11 matching lines...) Expand all Loading... | |
67 config_.reset(new vpx_codec_enc_cfg_t()); | 53 config_.reset(new vpx_codec_enc_cfg_t()); |
68 encoder_.reset(new vpx_codec_ctx_t()); | 54 encoder_.reset(new vpx_codec_ctx_t()); |
69 | 55 |
70 // Creating a wrapper to the image - setting image data to NULL. Actual | 56 // Creating a wrapper to the image - setting image data to NULL. Actual |
71 // pointer will be set during encode. Setting align to 1, as it is | 57 // pointer will be set during encode. Setting align to 1, as it is |
72 // meaningless (actual memory is not allocated). | 58 // meaningless (actual memory is not allocated). |
73 raw_image_ = vpx_img_wrap( | 59 raw_image_ = vpx_img_wrap( |
74 NULL, IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL); | 60 NULL, IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL); |
75 | 61 |
76 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { | 62 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { |
77 acked_frame_buffers_[i] = true; | 63 buffer_state_[i].frame_id = kStartFrameId; |
78 used_buffers_frame_id_[i] = kStartFrameId; | 64 buffer_state_[i].state = kBufferStartState; |
79 } | 65 } |
80 InitEncode(cast_config_.number_of_encode_threads); | 66 InitEncode(cast_config_.number_of_encode_threads); |
81 } | 67 } |
82 | 68 |
83 void Vp8Encoder::InitEncode(int number_of_encode_threads) { | 69 void Vp8Encoder::InitEncode(int number_of_encode_threads) { |
84 DCHECK(thread_checker_.CalledOnValidThread()); | 70 DCHECK(thread_checker_.CalledOnValidThread()); |
85 // Populate encoder configuration with default values. | 71 // Populate encoder configuration with default values. |
86 if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) { | 72 if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) { |
87 DCHECK(false) << "Invalid return value"; | 73 DCHECK(false) << "Invalid return value"; |
88 } | 74 } |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
153 vpx_codec_flags_t flags = 0; | 139 vpx_codec_flags_t flags = 0; |
154 if (key_frame_requested_) { | 140 if (key_frame_requested_) { |
155 flags = VPX_EFLAG_FORCE_KF; | 141 flags = VPX_EFLAG_FORCE_KF; |
156 // Self reference. | 142 // Self reference. |
157 latest_frame_id_to_reference = last_encoded_frame_id_ + 1; | 143 latest_frame_id_to_reference = last_encoded_frame_id_ + 1; |
158 // We can pick any buffer as buffer_to_update since we update | 144 // We can pick any buffer as buffer_to_update since we update |
159 // them all. | 145 // them all. |
160 buffer_to_update = kLastBuffer; | 146 buffer_to_update = kLastBuffer; |
161 } else { | 147 } else { |
162 // Reference all acked frames (buffers). | 148 // Reference all acked frames (buffers). |
163 latest_frame_id_to_reference = GetLatestFrameIdToReference(); | 149 latest_frame_id_to_reference = GetCodecReferenceFlags(&flags); |
164 GetCodecReferenceFlags(&flags); | |
165 buffer_to_update = GetNextBufferToUpdate(); | 150 buffer_to_update = GetNextBufferToUpdate(); |
166 GetCodecUpdateFlags(buffer_to_update, &flags); | 151 GetCodecUpdateFlags(buffer_to_update, &flags); |
167 } | 152 } |
168 | 153 |
169 // Note: The duration does not reflect the real time between frames. This is | 154 // Note: The duration does not reflect the real time between frames. This is |
170 // done to keep the encoder happy. | 155 // done to keep the encoder happy. |
171 // | 156 // |
172 // TODO(miu): This is a semi-hack. We should consider using | 157 // TODO(miu): This is a semi-hack. We should consider using |
173 // |video_frame->timestamp()| instead. | 158 // |video_frame->timestamp()| instead. |
174 uint32 duration = kVideoFrequency / cast_config_.max_frame_rate; | 159 uint32 duration = kVideoFrequency / cast_config_.max_frame_rate; |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
221 encoded_image->referenced_frame_id = latest_frame_id_to_reference; | 206 encoded_image->referenced_frame_id = latest_frame_id_to_reference; |
222 } | 207 } |
223 | 208 |
224 DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id | 209 DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id |
225 << ", sized:" << encoded_image->data.size(); | 210 << ", sized:" << encoded_image->data.size(); |
226 | 211 |
227 if (is_key_frame) { | 212 if (is_key_frame) { |
228 key_frame_requested_ = false; | 213 key_frame_requested_ = false; |
229 | 214 |
230 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { | 215 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { |
231 used_buffers_frame_id_[i] = encoded_image->frame_id; | 216 buffer_state_[i].state = kBufferSent; |
217 buffer_state_[i].frame_id = encoded_image->frame_id; | |
232 } | 218 } |
233 // We can pick any buffer as last_used_vp8_buffer_ since we update | |
234 // them all. | |
235 last_used_vp8_buffer_ = buffer_to_update; | |
236 } else { | 219 } else { |
237 if (buffer_to_update != kNoBuffer) { | 220 if (buffer_to_update != kNoBuffer) { |
238 acked_frame_buffers_[buffer_to_update] = false; | 221 buffer_state_[buffer_to_update].state = kBufferSent; |
239 used_buffers_frame_id_[buffer_to_update] = encoded_image->frame_id; | 222 buffer_state_[buffer_to_update].frame_id = encoded_image->frame_id; |
240 last_used_vp8_buffer_ = buffer_to_update; | |
241 } | 223 } |
242 } | 224 } |
243 return true; | 225 return true; |
244 } | 226 } |
245 | 227 |
246 void Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) { | 228 uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) { |
247 if (!use_multiple_video_buffers_) | 229 if (!use_multiple_video_buffers_) |
248 return; | 230 return last_encoded_frame_id_ + 1; |
249 | 231 |
250 // We need to reference something. | 232 uint32 latest_frame_to_reference = last_encoded_frame_id_ - 512; |
Alpha Left Google
2014/08/25 23:23:44
Why 512? Seems like this should be a constant and
hubbe
2014/08/27 04:14:03
Any number larger than 256 will work.
| |
251 DCHECK(acked_frame_buffers_[kAltRefBuffer] || | 233 // Reference all acked frames. |
252 acked_frame_buffers_[kGoldenBuffer] || | 234 // Probably only referencing the last one would be enough. |
253 acked_frame_buffers_[kLastBuffer]) | 235 // We also allow referencing the previous frame, even it has not |
254 << "Invalid state"; | 236 // been acked yet, but only if it was assigned to some buffer. |
255 | 237 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { |
256 if (!acked_frame_buffers_[kAltRefBuffer]) { | 238 if (buffer_state_[i].state == kBufferAcked) { |
Alpha Left Google
2014/08/25 23:23:44
This code reads to me that:
If buffer_state_[kAlt
hubbe
2014/08/27 04:14:03
Looks like I misplaced an else somewhere...
| |
257 *flags |= VP8_EFLAG_NO_REF_ARF; | 239 if (IsNewerFrameId(buffer_state_[i].frame_id, |
258 } | 240 latest_frame_to_reference)) { |
259 if (!acked_frame_buffers_[kGoldenBuffer]) { | 241 latest_frame_to_reference = buffer_state_[i].frame_id; |
260 *flags |= VP8_EFLAG_NO_REF_GF; | 242 } |
261 } | 243 switch (i) { |
262 if (!acked_frame_buffers_[kLastBuffer]) { | 244 case kAltRefBuffer: |
263 *flags |= VP8_EFLAG_NO_REF_LAST; | 245 *flags |= VP8_EFLAG_NO_REF_ARF; |
264 } | 246 break; |
265 } | 247 case kGoldenBuffer: |
266 | 248 *flags |= VP8_EFLAG_NO_REF_GF; |
267 uint32 Vp8Encoder::GetLatestFrameIdToReference() { | 249 break; |
268 if (!use_multiple_video_buffers_) | 250 case kLastBuffer: |
269 return last_encoded_frame_id_; | 251 *flags |= VP8_EFLAG_NO_REF_LAST; |
270 | 252 break; |
271 int latest_frame_id_to_reference = -1; | |
272 if (acked_frame_buffers_[kAltRefBuffer]) { | |
273 latest_frame_id_to_reference = used_buffers_frame_id_[kAltRefBuffer]; | |
274 } | |
275 if (acked_frame_buffers_[kGoldenBuffer]) { | |
276 if (latest_frame_id_to_reference == -1) { | |
277 latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer]; | |
278 } else { | |
279 if (IsNewerFrameId(used_buffers_frame_id_[kGoldenBuffer], | |
280 latest_frame_id_to_reference)) { | |
281 latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer]; | |
282 } | 253 } |
283 } | 254 } |
284 } | 255 } |
285 if (acked_frame_buffers_[kLastBuffer]) { | 256 |
286 if (latest_frame_id_to_reference == -1) { | 257 if (latest_frame_to_reference == last_encoded_frame_id_ - 512) { |
Alpha Left Google
2014/08/25 23:23:44
Why 512? Please add a comment.
hubbe
2014/08/27 04:14:03
It just needs to be the same as the 512 above, mad
| |
287 latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer]; | 258 // We have nothing to reference, it's kind of like a key frame, |
288 } else { | 259 // but doesn't reset buffers. |
289 if (IsNewerFrameId(used_buffers_frame_id_[kLastBuffer], | 260 latest_frame_to_reference = last_encoded_frame_id_ + 1; |
290 latest_frame_id_to_reference)) { | |
291 latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer]; | |
292 } | |
293 } | |
294 } | 261 } |
295 DCHECK(latest_frame_id_to_reference != -1) << "Invalid state"; | 262 |
296 return static_cast<uint32>(latest_frame_id_to_reference); | 263 return latest_frame_to_reference; |
297 } | 264 } |
298 | 265 |
299 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() { | 266 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() { |
300 if (!use_multiple_video_buffers_) | 267 if (!use_multiple_video_buffers_) |
301 return kNoBuffer; | 268 return kNoBuffer; |
302 | 269 |
303 // Update at most one buffer, except for key-frames. | 270 // Here are the rules for which buffer to select for update: |
271 // 1. If there is a buffer in state kStartState, use it. | |
272 // 2. If there is a buffer other than the oldest buffer | |
Alpha Left Google
2014/08/25 23:23:44
Please document the rationale for these conditions
hubbe
2014/08/27 04:14:03
Done.
| |
273 // which is Acked, use the oldest buffer. | |
274 // 3. If there are Sent buffers which are older than | |
275 // latest_acked_frame_, use the oldest one. | |
276 // 4. If all else fails, just overwrite the newest buffer, | |
277 // but no more than 3 times in a row. | |
304 | 278 |
305 Vp8Buffers buffer_to_update = kNoBuffer; | 279 // Buffers, sorted from oldest frame to newest. |
306 if (number_of_repeated_buffers_ < max_number_of_repeated_buffers_in_a_row_) { | 280 Vp8Encoder::Vp8Buffers buffers[kNumberOfVp8VideoBuffers]; |
307 // TODO(pwestin): experiment with this. The issue with only this change is | 281 |
308 // that we can end up with only 4 frames in flight when we expect 6. | 282 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { |
309 // buffer_to_update = last_used_vp8_buffer_; | 283 Vp8Encoder::Vp8Buffers buffer = static_cast<Vp8Encoder::Vp8Buffers>(i); |
310 buffer_to_update = kNoBuffer; | 284 |
311 ++number_of_repeated_buffers_; | 285 // Rule 1 |
312 } else { | 286 if (buffer_state_[buffer].state == kBufferStartState) { |
313 number_of_repeated_buffers_ = 0; | 287 undroppable_frames_ = 0; |
314 switch (last_used_vp8_buffer_) { | 288 return buffer; |
315 case kAltRefBuffer: | 289 } |
316 buffer_to_update = kLastBuffer; | 290 buffers[buffer] = buffer; |
317 VLOG(1) << "VP8 update last buffer"; | 291 } |
318 break; | 292 |
319 case kLastBuffer: | 293 // Sorting three elements with selection sort. |
320 buffer_to_update = kGoldenBuffer; | 294 for (int i = 0; i < kNumberOfVp8VideoBuffers - 1; i++) { |
321 VLOG(1) << "VP8 update golden buffer"; | 295 for (int j = i + 1; j < kNumberOfVp8VideoBuffers; j++) { |
322 break; | 296 if (IsOlderFrameId(buffer_state_[buffers[j]].frame_id, |
323 case kGoldenBuffer: | 297 buffer_state_[buffers[i]].frame_id)) { |
324 buffer_to_update = kAltRefBuffer; | 298 std::swap(buffers[i], buffers[j]); |
325 VLOG(1) << "VP8 update alt-ref buffer"; | 299 } |
326 break; | |
327 case kNoBuffer: | |
328 DCHECK(false) << "Invalid state"; | |
329 break; | |
330 } | 300 } |
331 } | 301 } |
332 return buffer_to_update; | 302 |
303 // Rule 2 | |
304 if (buffer_state_[buffers[1]].state == kBufferAcked || | |
305 buffer_state_[buffers[2]].state == kBufferAcked) { | |
306 undroppable_frames_ = 0; | |
307 return buffers[0]; | |
308 } | |
309 | |
310 // Rule 3 | |
311 for (int i = 0; i < kNumberOfVp8VideoBuffers; i++) { | |
312 if (buffer_state_[buffers[i]].state == kBufferSent && | |
313 IsOlderFrameId(buffer_state_[buffers[i]].frame_id, | |
314 last_acked_frame_id_)) { | |
315 undroppable_frames_ = 0; | |
316 return buffers[i]; | |
317 } | |
318 } | |
319 | |
320 // Rule 4 | |
321 if (undroppable_frames_ >= 3) { | |
322 undroppable_frames_ = 0; | |
323 return kNoBuffer; | |
324 } else { | |
325 undroppable_frames_++; | |
326 return buffers[kNumberOfVp8VideoBuffers - 1]; | |
327 } | |
333 } | 328 } |
334 | 329 |
335 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update, | 330 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update, |
336 vpx_codec_flags_t* flags) { | 331 vpx_codec_flags_t* flags) { |
337 if (!use_multiple_video_buffers_) | 332 if (!use_multiple_video_buffers_) |
338 return; | 333 return; |
339 | 334 |
340 // Update at most one buffer, except for key-frames. | 335 // Update at most one buffer, except for key-frames. |
341 switch (buffer_to_update) { | 336 switch (buffer_to_update) { |
342 case kAltRefBuffer: | 337 case kAltRefBuffer: |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
374 } | 369 } |
375 } | 370 } |
376 | 371 |
377 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) { | 372 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) { |
378 DCHECK(thread_checker_.CalledOnValidThread()); | 373 DCHECK(thread_checker_.CalledOnValidThread()); |
379 if (!use_multiple_video_buffers_) | 374 if (!use_multiple_video_buffers_) |
380 return; | 375 return; |
381 | 376 |
382 VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id); | 377 VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id); |
383 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { | 378 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { |
384 if (frame_id == used_buffers_frame_id_[i]) { | 379 if (frame_id == buffer_state_[i].frame_id) { |
385 acked_frame_buffers_[i] = true; | 380 buffer_state_[i].state = kBufferAcked; |
381 break; | |
382 } | |
383 if (IsOlderFrameId(last_acked_frame_id_, frame_id)) { | |
Alpha Left Google
2014/08/25 23:23:44
Why is this inside the for loop? I don't understan
hubbe
2014/08/27 04:14:03
Messed up merge, fixed.
| |
384 last_acked_frame_id_ = frame_id; | |
386 } | 385 } |
387 } | 386 } |
388 } | 387 } |
389 | 388 |
390 void Vp8Encoder::GenerateKeyFrame() { | 389 void Vp8Encoder::GenerateKeyFrame() { |
391 DCHECK(thread_checker_.CalledOnValidThread()); | 390 DCHECK(thread_checker_.CalledOnValidThread()); |
392 key_frame_requested_ = true; | 391 key_frame_requested_ = true; |
393 } | 392 } |
394 | 393 |
395 // Calculate the max size of the key frame relative to a normal delta frame. | 394 // Calculate the max size of the key frame relative to a normal delta frame. |
396 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const { | 395 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const { |
397 // Set max to the optimal buffer level (normalized by target BR), | 396 // Set max to the optimal buffer level (normalized by target BR), |
398 // and scaled by a scale_parameter. | 397 // and scaled by a scale_parameter. |
399 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps]. | 398 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps]. |
400 // This values is presented in percentage of perFrameBw: | 399 // This values is presented in percentage of perFrameBw: |
401 // perFrameBw = targetBR[Kbps] * 1000 / frameRate. | 400 // perFrameBw = targetBR[Kbps] * 1000 / frameRate. |
402 // The target in % is as follows: | 401 // The target in % is as follows: |
403 | 402 |
404 float scale_parameter = 0.5; | 403 float scale_parameter = 0.5; |
405 uint32 target_pct = optimal_buffer_size_ms * scale_parameter * | 404 uint32 target_pct = optimal_buffer_size_ms * scale_parameter * |
406 cast_config_.max_frame_rate / 10; | 405 cast_config_.max_frame_rate / 10; |
407 | 406 |
408 // Don't go below 3 times the per frame bandwidth. | 407 // Don't go below 3 times the per frame bandwidth. |
409 return std::max(target_pct, kMinIntra); | 408 return std::max(target_pct, kMinIntra); |
410 } | 409 } |
411 | 410 |
412 } // namespace cast | 411 } // namespace cast |
413 } // namespace media | 412 } // namespace media |
OLD | NEW |