Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(676)

Side by Side Diff: media/cast/sender/vp8_encoder.cc

Issue 339743002: Cast: Make vp8 3-buffer mode work (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: merge, fix, cleanup, rince, repeat Created 6 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2014 The Chromium Authors. All rights reserved. 1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "media/cast/sender/vp8_encoder.h" 5 #include "media/cast/sender/vp8_encoder.h"
6 6
7 #include <vector> 7 #include <vector>
8 8
9 #include "base/logging.h" 9 #include "base/logging.h"
10 #include "media/base/video_frame.h" 10 #include "media/base/video_frame.h"
11 #include "media/cast/cast_defines.h" 11 #include "media/cast/cast_defines.h"
12 #include "media/cast/net/cast_transport_config.h" 12 #include "media/cast/net/cast_transport_config.h"
13 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" 13 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
14 14
15 namespace media { 15 namespace media {
16 namespace cast { 16 namespace cast {
17 17
18 static const uint32 kMinIntra = 300; 18 static const uint32 kMinIntra = 300;
19 19
20 static int ComputeMaxNumOfRepeatedBuffes(int max_unacked_frames) {
21 if (max_unacked_frames > kNumberOfVp8VideoBuffers)
22 return (max_unacked_frames - 1) / kNumberOfVp8VideoBuffers;
23
24 return 0;
25 }
26
27 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config, 20 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config,
28 int max_unacked_frames) 21 int max_unacked_frames)
29 : cast_config_(video_config), 22 : cast_config_(video_config),
30 use_multiple_video_buffers_( 23 use_multiple_video_buffers_(
31 cast_config_.max_number_of_video_buffers_used == 24 cast_config_.max_number_of_video_buffers_used ==
32 kNumberOfVp8VideoBuffers), 25 kNumberOfVp8VideoBuffers),
33 max_number_of_repeated_buffers_in_a_row_(
34 ComputeMaxNumOfRepeatedBuffes(max_unacked_frames)),
35 key_frame_requested_(true), 26 key_frame_requested_(true),
36 first_frame_received_(false), 27 first_frame_received_(false),
37 last_encoded_frame_id_(kStartFrameId), 28 last_encoded_frame_id_(kStartFrameId),
38 number_of_repeated_buffers_(0) { 29 last_acked_frame_id_(kStartFrameId),
39 // TODO(pwestin): we need to figure out how to synchronize the acking with the 30 frame_id_to_reference_(kStartFrameId - 1),
40 // internal state of the encoder, ideally the encoder will tell if we can 31 undroppable_frames_(0) {
41 // send another frame.
42 DCHECK(!use_multiple_video_buffers_ ||
43 max_number_of_repeated_buffers_in_a_row_ == 0)
44 << "Invalid config";
45
46 // VP8 have 3 buffers available for prediction, with 32 // VP8 have 3 buffers available for prediction, with
47 // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency 33 // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
48 // however in this mode we can not skip frames in the receiver to catch up 34 // however in this mode we can not skip frames in the receiver to catch up
49 // after a temporary network outage; with max_number_of_video_buffers_used 35 // after a temporary network outage; with max_number_of_video_buffers_used
50 // set to 3 we allow 2 frames to be skipped by the receiver without error 36 // set to 3 we allow 2 frames to be skipped by the receiver without error
51 // propagation. 37 // propagation.
52 DCHECK(cast_config_.max_number_of_video_buffers_used == 1 || 38 DCHECK(cast_config_.max_number_of_video_buffers_used == 1 ||
53 cast_config_.max_number_of_video_buffers_used == 39 cast_config_.max_number_of_video_buffers_used ==
54 kNumberOfVp8VideoBuffers) 40 kNumberOfVp8VideoBuffers)
55 << "Invalid argument"; 41 << "Invalid argument";
(...skipping 11 matching lines...) Expand all
67 config_.reset(new vpx_codec_enc_cfg_t()); 53 config_.reset(new vpx_codec_enc_cfg_t());
68 encoder_.reset(new vpx_codec_ctx_t()); 54 encoder_.reset(new vpx_codec_ctx_t());
69 55
70 // Creating a wrapper to the image - setting image data to NULL. Actual 56 // Creating a wrapper to the image - setting image data to NULL. Actual
71 // pointer will be set during encode. Setting align to 1, as it is 57 // pointer will be set during encode. Setting align to 1, as it is
72 // meaningless (actual memory is not allocated). 58 // meaningless (actual memory is not allocated).
73 raw_image_ = vpx_img_wrap( 59 raw_image_ = vpx_img_wrap(
74 NULL, IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL); 60 NULL, IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL);
75 61
76 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 62 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
77 acked_frame_buffers_[i] = true; 63 buffer_state_[i].frame_id = kStartFrameId;
78 used_buffers_frame_id_[i] = kStartFrameId; 64 buffer_state_[i].state = kBufferStartState;
79 } 65 }
80 InitEncode(cast_config_.number_of_encode_threads); 66 InitEncode(cast_config_.number_of_encode_threads);
81 } 67 }
82 68
83 void Vp8Encoder::InitEncode(int number_of_encode_threads) { 69 void Vp8Encoder::InitEncode(int number_of_encode_threads) {
84 DCHECK(thread_checker_.CalledOnValidThread()); 70 DCHECK(thread_checker_.CalledOnValidThread());
85 // Populate encoder configuration with default values. 71 // Populate encoder configuration with default values.
86 if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) { 72 if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) {
87 DCHECK(false) << "Invalid return value"; 73 DCHECK(false) << "Invalid return value";
88 } 74 }
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
153 vpx_codec_flags_t flags = 0; 139 vpx_codec_flags_t flags = 0;
154 if (key_frame_requested_) { 140 if (key_frame_requested_) {
155 flags = VPX_EFLAG_FORCE_KF; 141 flags = VPX_EFLAG_FORCE_KF;
156 // Self reference. 142 // Self reference.
157 latest_frame_id_to_reference = last_encoded_frame_id_ + 1; 143 latest_frame_id_to_reference = last_encoded_frame_id_ + 1;
158 // We can pick any buffer as buffer_to_update since we update 144 // We can pick any buffer as buffer_to_update since we update
159 // them all. 145 // them all.
160 buffer_to_update = kLastBuffer; 146 buffer_to_update = kLastBuffer;
161 } else { 147 } else {
162 // Reference all acked frames (buffers). 148 // Reference all acked frames (buffers).
163 latest_frame_id_to_reference = GetLatestFrameIdToReference(); 149 latest_frame_id_to_reference = GetCodecReferenceFlags(&flags);
164 GetCodecReferenceFlags(&flags);
165 buffer_to_update = GetNextBufferToUpdate(); 150 buffer_to_update = GetNextBufferToUpdate();
166 GetCodecUpdateFlags(buffer_to_update, &flags); 151 GetCodecUpdateFlags(buffer_to_update, &flags);
167 } 152 }
168 153
169 // Note: The duration does not reflect the real time between frames. This is 154 // Note: The duration does not reflect the real time between frames. This is
170 // done to keep the encoder happy. 155 // done to keep the encoder happy.
171 // 156 //
172 // TODO(miu): This is a semi-hack. We should consider using 157 // TODO(miu): This is a semi-hack. We should consider using
173 // |video_frame->timestamp()| instead. 158 // |video_frame->timestamp()| instead.
174 uint32 duration = kVideoFrequency / cast_config_.max_frame_rate; 159 uint32 duration = kVideoFrequency / cast_config_.max_frame_rate;
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
221 encoded_image->referenced_frame_id = latest_frame_id_to_reference; 206 encoded_image->referenced_frame_id = latest_frame_id_to_reference;
222 } 207 }
223 208
224 DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id 209 DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id
225 << ", sized:" << encoded_image->data.size(); 210 << ", sized:" << encoded_image->data.size();
226 211
227 if (is_key_frame) { 212 if (is_key_frame) {
228 key_frame_requested_ = false; 213 key_frame_requested_ = false;
229 214
230 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 215 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
231 used_buffers_frame_id_[i] = encoded_image->frame_id; 216 buffer_state_[i].state = kBufferSent;
217 buffer_state_[i].frame_id = encoded_image->frame_id;
232 } 218 }
233 // We can pick any buffer as last_used_vp8_buffer_ since we update
234 // them all.
235 last_used_vp8_buffer_ = buffer_to_update;
236 } else { 219 } else {
237 if (buffer_to_update != kNoBuffer) { 220 if (buffer_to_update != kNoBuffer) {
238 acked_frame_buffers_[buffer_to_update] = false; 221 buffer_state_[buffer_to_update].state = kBufferSent;
239 used_buffers_frame_id_[buffer_to_update] = encoded_image->frame_id; 222 buffer_state_[buffer_to_update].frame_id = encoded_image->frame_id;
240 last_used_vp8_buffer_ = buffer_to_update;
241 } 223 }
242 } 224 }
243 return true; 225 return true;
244 } 226 }
245 227
246 void Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) { 228 uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
247 if (!use_multiple_video_buffers_) 229 if (!use_multiple_video_buffers_)
248 return; 230 return last_encoded_frame_id_ + 1;
249 231
250 // We need to reference something. 232 uint32 latest_frame_to_reference = last_encoded_frame_id_ - 512;
Alpha Left Google 2014/08/25 23:23:44 Why 512? Seems like this should be a constant and
hubbe 2014/08/27 04:14:03 Any number larger than 256 will work.
251 DCHECK(acked_frame_buffers_[kAltRefBuffer] || 233 // Reference all acked frames.
252 acked_frame_buffers_[kGoldenBuffer] || 234 // Probably only referencing the last one would be enough.
253 acked_frame_buffers_[kLastBuffer]) 235 // We also allow referencing the previous frame, even it has not
254 << "Invalid state"; 236 // been acked yet, but only if it was assigned to some buffer.
255 237 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
256 if (!acked_frame_buffers_[kAltRefBuffer]) { 238 if (buffer_state_[i].state == kBufferAcked) {
Alpha Left Google 2014/08/25 23:23:44 This code reads to me that: If buffer_state_[kAlt
hubbe 2014/08/27 04:14:03 Looks like I misplaced an else somewhere...
257 *flags |= VP8_EFLAG_NO_REF_ARF; 239 if (IsNewerFrameId(buffer_state_[i].frame_id,
258 } 240 latest_frame_to_reference)) {
259 if (!acked_frame_buffers_[kGoldenBuffer]) { 241 latest_frame_to_reference = buffer_state_[i].frame_id;
260 *flags |= VP8_EFLAG_NO_REF_GF; 242 }
261 } 243 switch (i) {
262 if (!acked_frame_buffers_[kLastBuffer]) { 244 case kAltRefBuffer:
263 *flags |= VP8_EFLAG_NO_REF_LAST; 245 *flags |= VP8_EFLAG_NO_REF_ARF;
264 } 246 break;
265 } 247 case kGoldenBuffer:
266 248 *flags |= VP8_EFLAG_NO_REF_GF;
267 uint32 Vp8Encoder::GetLatestFrameIdToReference() { 249 break;
268 if (!use_multiple_video_buffers_) 250 case kLastBuffer:
269 return last_encoded_frame_id_; 251 *flags |= VP8_EFLAG_NO_REF_LAST;
270 252 break;
271 int latest_frame_id_to_reference = -1;
272 if (acked_frame_buffers_[kAltRefBuffer]) {
273 latest_frame_id_to_reference = used_buffers_frame_id_[kAltRefBuffer];
274 }
275 if (acked_frame_buffers_[kGoldenBuffer]) {
276 if (latest_frame_id_to_reference == -1) {
277 latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
278 } else {
279 if (IsNewerFrameId(used_buffers_frame_id_[kGoldenBuffer],
280 latest_frame_id_to_reference)) {
281 latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
282 } 253 }
283 } 254 }
284 } 255 }
285 if (acked_frame_buffers_[kLastBuffer]) { 256
286 if (latest_frame_id_to_reference == -1) { 257 if (latest_frame_to_reference == last_encoded_frame_id_ - 512) {
Alpha Left Google 2014/08/25 23:23:44 Why 512? Please add a comment.
hubbe 2014/08/27 04:14:03 It just needs to be the same as the 512 above, mad
287 latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer]; 258 // We have nothing to reference, it's kind of like a key frame,
288 } else { 259 // but doesn't reset buffers.
289 if (IsNewerFrameId(used_buffers_frame_id_[kLastBuffer], 260 latest_frame_to_reference = last_encoded_frame_id_ + 1;
290 latest_frame_id_to_reference)) {
291 latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
292 }
293 }
294 } 261 }
295 DCHECK(latest_frame_id_to_reference != -1) << "Invalid state"; 262
296 return static_cast<uint32>(latest_frame_id_to_reference); 263 return latest_frame_to_reference;
297 } 264 }
298 265
299 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() { 266 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() {
300 if (!use_multiple_video_buffers_) 267 if (!use_multiple_video_buffers_)
301 return kNoBuffer; 268 return kNoBuffer;
302 269
303 // Update at most one buffer, except for key-frames. 270 // Here are the rules for which buffer to select for update:
271 // 1. If there is a buffer in state kStartState, use it.
272 // 2. If there is a buffer other than the oldest buffer
Alpha Left Google 2014/08/25 23:23:44 Please document the rationale for these conditions
hubbe 2014/08/27 04:14:03 Done.
273 // which is Acked, use the oldest buffer.
274 // 3. If there are Sent buffers which are older than
275 // latest_acked_frame_, use the oldest one.
276 // 4. If all else fails, just overwrite the newest buffer,
277 // but no more than 3 times in a row.
304 278
305 Vp8Buffers buffer_to_update = kNoBuffer; 279 // Buffers, sorted from oldest frame to newest.
306 if (number_of_repeated_buffers_ < max_number_of_repeated_buffers_in_a_row_) { 280 Vp8Encoder::Vp8Buffers buffers[kNumberOfVp8VideoBuffers];
307 // TODO(pwestin): experiment with this. The issue with only this change is 281
308 // that we can end up with only 4 frames in flight when we expect 6. 282 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
309 // buffer_to_update = last_used_vp8_buffer_; 283 Vp8Encoder::Vp8Buffers buffer = static_cast<Vp8Encoder::Vp8Buffers>(i);
310 buffer_to_update = kNoBuffer; 284
311 ++number_of_repeated_buffers_; 285 // Rule 1
312 } else { 286 if (buffer_state_[buffer].state == kBufferStartState) {
313 number_of_repeated_buffers_ = 0; 287 undroppable_frames_ = 0;
314 switch (last_used_vp8_buffer_) { 288 return buffer;
315 case kAltRefBuffer: 289 }
316 buffer_to_update = kLastBuffer; 290 buffers[buffer] = buffer;
317 VLOG(1) << "VP8 update last buffer"; 291 }
318 break; 292
319 case kLastBuffer: 293 // Sorting three elements with selection sort.
320 buffer_to_update = kGoldenBuffer; 294 for (int i = 0; i < kNumberOfVp8VideoBuffers - 1; i++) {
321 VLOG(1) << "VP8 update golden buffer"; 295 for (int j = i + 1; j < kNumberOfVp8VideoBuffers; j++) {
322 break; 296 if (IsOlderFrameId(buffer_state_[buffers[j]].frame_id,
323 case kGoldenBuffer: 297 buffer_state_[buffers[i]].frame_id)) {
324 buffer_to_update = kAltRefBuffer; 298 std::swap(buffers[i], buffers[j]);
325 VLOG(1) << "VP8 update alt-ref buffer"; 299 }
326 break;
327 case kNoBuffer:
328 DCHECK(false) << "Invalid state";
329 break;
330 } 300 }
331 } 301 }
332 return buffer_to_update; 302
303 // Rule 2
304 if (buffer_state_[buffers[1]].state == kBufferAcked ||
305 buffer_state_[buffers[2]].state == kBufferAcked) {
306 undroppable_frames_ = 0;
307 return buffers[0];
308 }
309
310 // Rule 3
311 for (int i = 0; i < kNumberOfVp8VideoBuffers; i++) {
312 if (buffer_state_[buffers[i]].state == kBufferSent &&
313 IsOlderFrameId(buffer_state_[buffers[i]].frame_id,
314 last_acked_frame_id_)) {
315 undroppable_frames_ = 0;
316 return buffers[i];
317 }
318 }
319
320 // Rule 4
321 if (undroppable_frames_ >= 3) {
322 undroppable_frames_ = 0;
323 return kNoBuffer;
324 } else {
325 undroppable_frames_++;
326 return buffers[kNumberOfVp8VideoBuffers - 1];
327 }
333 } 328 }
334 329
335 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update, 330 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update,
336 vpx_codec_flags_t* flags) { 331 vpx_codec_flags_t* flags) {
337 if (!use_multiple_video_buffers_) 332 if (!use_multiple_video_buffers_)
338 return; 333 return;
339 334
340 // Update at most one buffer, except for key-frames. 335 // Update at most one buffer, except for key-frames.
341 switch (buffer_to_update) { 336 switch (buffer_to_update) {
342 case kAltRefBuffer: 337 case kAltRefBuffer:
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
374 } 369 }
375 } 370 }
376 371
377 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) { 372 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
378 DCHECK(thread_checker_.CalledOnValidThread()); 373 DCHECK(thread_checker_.CalledOnValidThread());
379 if (!use_multiple_video_buffers_) 374 if (!use_multiple_video_buffers_)
380 return; 375 return;
381 376
382 VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id); 377 VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
383 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) { 378 for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
384 if (frame_id == used_buffers_frame_id_[i]) { 379 if (frame_id == buffer_state_[i].frame_id) {
385 acked_frame_buffers_[i] = true; 380 buffer_state_[i].state = kBufferAcked;
381 break;
382 }
383 if (IsOlderFrameId(last_acked_frame_id_, frame_id)) {
Alpha Left Google 2014/08/25 23:23:44 Why is this inside the for loop? I don't understan
hubbe 2014/08/27 04:14:03 Messed up merge, fixed.
384 last_acked_frame_id_ = frame_id;
386 } 385 }
387 } 386 }
388 } 387 }
389 388
390 void Vp8Encoder::GenerateKeyFrame() { 389 void Vp8Encoder::GenerateKeyFrame() {
391 DCHECK(thread_checker_.CalledOnValidThread()); 390 DCHECK(thread_checker_.CalledOnValidThread());
392 key_frame_requested_ = true; 391 key_frame_requested_ = true;
393 } 392 }
394 393
395 // Calculate the max size of the key frame relative to a normal delta frame. 394 // Calculate the max size of the key frame relative to a normal delta frame.
396 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const { 395 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const {
397 // Set max to the optimal buffer level (normalized by target BR), 396 // Set max to the optimal buffer level (normalized by target BR),
398 // and scaled by a scale_parameter. 397 // and scaled by a scale_parameter.
399 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps]. 398 // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
400 // This values is presented in percentage of perFrameBw: 399 // This values is presented in percentage of perFrameBw:
401 // perFrameBw = targetBR[Kbps] * 1000 / frameRate. 400 // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
402 // The target in % is as follows: 401 // The target in % is as follows:
403 402
404 float scale_parameter = 0.5; 403 float scale_parameter = 0.5;
405 uint32 target_pct = optimal_buffer_size_ms * scale_parameter * 404 uint32 target_pct = optimal_buffer_size_ms * scale_parameter *
406 cast_config_.max_frame_rate / 10; 405 cast_config_.max_frame_rate / 10;
407 406
408 // Don't go below 3 times the per frame bandwidth. 407 // Don't go below 3 times the per frame bandwidth.
409 return std::max(target_pct, kMinIntra); 408 return std::max(target_pct, kMinIntra);
410 } 409 }
411 410
412 } // namespace cast 411 } // namespace cast
413 } // namespace media 412 } // namespace media
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698