OLD | NEW |
(Empty) | |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #include "media/filters/video_renderer_algorithm.h" |
| 6 |
| 7 #include <algorithm> |
| 8 #include <limits> |
| 9 |
| 10 namespace media { |
| 11 |
| 12 // The number of frames to store for moving average calculations. Value picked |
| 13 // after experimenting with playback of various local media and YouTube clips. |
| 14 const int kMovingAverageSamples = 25; |
| 15 |
| 16 VideoRendererAlgorithm::ReadyFrame::ReadyFrame( |
| 17 const scoped_refptr<VideoFrame>& ready_frame) |
| 18 : frame(ready_frame), |
| 19 ideal_render_count(0), |
| 20 render_count(0), |
| 21 drop_count(0) { |
| 22 } |
| 23 |
| 24 VideoRendererAlgorithm::ReadyFrame::~ReadyFrame() { |
| 25 } |
| 26 |
| 27 bool VideoRendererAlgorithm::ReadyFrame::operator<( |
| 28 const ReadyFrame& other) const { |
| 29 return frame->timestamp() < other.frame->timestamp(); |
| 30 } |
| 31 |
| 32 VideoRendererAlgorithm::VideoRendererAlgorithm( |
| 33 const TimeConverterCB& time_converter_cb) |
| 34 : cadence_estimator_(base::TimeDelta::FromSeconds( |
| 35 kMinimumAcceptableTimeBetweenGlitchesSecs)), |
| 36 time_converter_cb_(time_converter_cb), |
| 37 frame_duration_calculator_(kMovingAverageSamples), |
| 38 frame_dropping_disabled_(false) { |
| 39 DCHECK(!time_converter_cb_.is_null()); |
| 40 Reset(); |
| 41 } |
| 42 |
| 43 VideoRendererAlgorithm::~VideoRendererAlgorithm() { |
| 44 } |
| 45 |
| 46 scoped_refptr<VideoFrame> VideoRendererAlgorithm::Render( |
| 47 base::TimeTicks deadline_min, |
| 48 base::TimeTicks deadline_max, |
| 49 size_t* frames_dropped) { |
| 50 DCHECK_LT(deadline_min, deadline_max); |
| 51 |
| 52 if (frame_queue_.empty()) |
| 53 return nullptr; |
| 54 |
| 55 if (frames_dropped) |
| 56 *frames_dropped = 0; |
| 57 |
| 58 // Once Render() is called |last_frame_index_| has meaning and should thus be |
| 59 // preserved even if better frames come in before it due to out of order |
| 60 // timestamps. |
| 61 have_rendered_frames_ = true; |
| 62 |
| 63 // Step 1: Update the current render interval for subroutines. |
| 64 render_interval_ = deadline_max - deadline_min; |
| 65 |
| 66 // Step 2: Figure out if any intervals have been skipped since the last call |
| 67 // to Render(). If so, we assume the last frame provided was rendered during |
| 68 // those intervals and adjust its render count appropriately. |
| 69 AccountForMissedIntervals(deadline_min, deadline_max); |
| 70 last_deadline_max_ = deadline_max; |
| 71 |
| 72 // Step 3: Update the wall clock timestamps and frame duration estimates for |
| 73 // all frames currently in the |frame_queue_|. |
| 74 if (!UpdateFrameStatistics()) { |
| 75 DVLOG(2) << "Failed to update frame statistics."; |
| 76 |
| 77 ReadyFrame& ready_frame = frame_queue_[last_frame_index_]; |
| 78 DCHECK(ready_frame.frame); |
| 79 |
| 80 // If duration is unknown, we don't have enough frames to make a good guess |
| 81 // about which frame to use, so always choose the first. |
| 82 if (average_frame_duration_ == base::TimeDelta() && |
| 83 !ready_frame.wall_clock_time.is_null()) { |
| 84 ++ready_frame.render_count; |
| 85 } |
| 86 |
| 87 return ready_frame.frame; |
| 88 } |
| 89 |
| 90 DCHECK_GT(average_frame_duration_, base::TimeDelta()); |
| 91 |
| 92 base::TimeDelta selected_frame_drift; |
| 93 |
| 94 // Step 4: Attempt to find the best frame by cadence. |
| 95 int frame_to_render = FindBestFrameByCadence(); |
| 96 if (frame_to_render >= 0) { |
| 97 selected_frame_drift = |
| 98 CalculateAbsoluteDriftForFrame(deadline_min, frame_to_render); |
| 99 } |
| 100 |
| 101 // Step 5: If no frame could be found by cadence or the selected frame exceeds |
| 102 // acceptable drift, try to find the best frame by coverage of the deadline. |
| 103 if (frame_to_render < 0 || selected_frame_drift > max_acceptable_drift_) { |
| 104 int second_best_by_coverage = -1; |
| 105 const int best_by_coverage = FindBestFrameByCoverage( |
| 106 deadline_min, deadline_max, &second_best_by_coverage); |
| 107 |
| 108 // If the frame was previously selected based on cadence, we're only here |
| 109 // because the drift is too large, so even if the cadence frame has the best |
| 110 // coverage, fallback to the second best by coverage if it has better drift. |
| 111 if (frame_to_render == best_by_coverage && second_best_by_coverage >= 0 && |
| 112 CalculateAbsoluteDriftForFrame(deadline_min, second_best_by_coverage) <= |
| 113 selected_frame_drift) { |
| 114 frame_to_render = second_best_by_coverage; |
| 115 } else { |
| 116 frame_to_render = best_by_coverage; |
| 117 } |
| 118 |
| 119 if (frame_to_render >= 0) { |
| 120 selected_frame_drift = |
| 121 CalculateAbsoluteDriftForFrame(deadline_min, frame_to_render); |
| 122 } |
| 123 } |
| 124 |
| 125 // Step 6: If _still_ no frame could be found by coverage, try to choose the |
| 126 // least crappy option based on the drift from the deadline. If we're here the |
| 127 // selection is going to be bad because it means no suitable frame has any |
| 128 // coverage of the deadline interval. |
| 129 if (frame_to_render < 0 || selected_frame_drift > max_acceptable_drift_) |
| 130 frame_to_render = FindBestFrameByDrift(deadline_min, &selected_frame_drift); |
| 131 |
| 132 last_render_had_glitch_ = selected_frame_drift > max_acceptable_drift_; |
| 133 DVLOG_IF(2, last_render_had_glitch_) |
| 134 << "Frame drift is too far: " << selected_frame_drift.InMillisecondsF() |
| 135 << "ms"; |
| 136 |
| 137 DCHECK_GE(frame_to_render, 0); |
| 138 |
| 139 // Drop some debugging information if a frame had poor cadence. |
| 140 if (cadence_estimator_.has_cadence()) { |
| 141 const ReadyFrame& last_frame_info = frame_queue_[last_frame_index_]; |
| 142 if (static_cast<size_t>(frame_to_render) != last_frame_index_ && |
| 143 last_frame_info.render_count < last_frame_info.ideal_render_count) { |
| 144 last_render_had_glitch_ = true; |
| 145 DVLOG(2) << "Under-rendered frame " << last_frame_info.frame->timestamp() |
| 146 << "; only " << last_frame_info.render_count |
| 147 << " times instead of " << last_frame_info.ideal_render_count; |
| 148 } else if (static_cast<size_t>(frame_to_render) == last_frame_index_ && |
| 149 last_frame_info.render_count >= |
| 150 last_frame_info.ideal_render_count) { |
| 151 DVLOG(2) << "Over-rendered frame " << last_frame_info.frame->timestamp() |
| 152 << "; rendered " << last_frame_info.render_count + 1 |
| 153 << " times instead of " << last_frame_info.ideal_render_count; |
| 154 last_render_had_glitch_ = true; |
| 155 } |
| 156 } |
| 157 |
| 158 // Step 7: Drop frames which occur prior to the frame to be rendered. If any |
| 159 // frame has a zero render count it should be reported as dropped. |
| 160 if (frame_to_render > 0) { |
| 161 if (frames_dropped) { |
| 162 for (int i = 0; i < frame_to_render; ++i) { |
| 163 const ReadyFrame& frame = frame_queue_[i]; |
| 164 if (frame.render_count != frame.drop_count) |
| 165 continue; |
| 166 |
| 167 // If frame dropping is disabled, ignore the results of the algorithm |
| 168 // and return the earliest unrendered frame. |
| 169 if (frame_dropping_disabled_) { |
| 170 frame_to_render = i; |
| 171 break; |
| 172 } |
| 173 |
| 174 DVLOG(2) << "Dropping unrendered (or always dropped) frame " |
| 175 << frame.frame->timestamp() |
| 176 << ", wall clock: " << frame.wall_clock_time.ToInternalValue() |
| 177 << " (" << frame.render_count << ", " << frame.drop_count |
| 178 << ")"; |
| 179 ++(*frames_dropped); |
| 180 if (!cadence_estimator_.has_cadence() || frame.ideal_render_count) |
| 181 last_render_had_glitch_ = true; |
| 182 } |
| 183 } |
| 184 |
| 185 frame_queue_.erase(frame_queue_.begin(), |
| 186 frame_queue_.begin() + frame_to_render); |
| 187 } |
| 188 |
| 189 if (last_render_had_glitch_) { |
| 190 DVLOG(2) << "Deadline: [" << deadline_min.ToInternalValue() << ", " |
| 191 << deadline_max.ToInternalValue() |
| 192 << "], Interval: " << render_interval_.InMicroseconds() |
| 193 << ", Duration: " << average_frame_duration_.InMicroseconds(); |
| 194 } |
| 195 |
| 196 // Step 8: Congratulations, the frame selection gauntlet has been passed! |
| 197 last_frame_index_ = 0; |
| 198 ++frame_queue_.front().render_count; |
| 199 DCHECK(frame_queue_.front().frame); |
| 200 return frame_queue_.front().frame; |
| 201 } |
| 202 |
| 203 size_t VideoRendererAlgorithm::RemoveExpiredFrames(base::TimeTicks deadline) { |
| 204 // Update |last_deadline_max_| if it's no longer accurate; this should always |
| 205 // be done or EffectiveFramesQueued() may never expire the last frame. |
| 206 if (deadline > last_deadline_max_) |
| 207 last_deadline_max_ = deadline; |
| 208 |
| 209 if (!UpdateFrameStatistics() || frame_queue_.size() < 2) |
| 210 return 0; |
| 211 |
| 212 DCHECK_GT(average_frame_duration_, base::TimeDelta()); |
| 213 |
| 214 // Finds and removes all frames which are too old to be used; I.e., the end of |
| 215 // their render interval is further than |max_acceptable_drift_| from the |
| 216 // given |deadline|. |
| 217 size_t frames_to_expire = 0; |
| 218 const base::TimeTicks minimum_frame_time = |
| 219 deadline - max_acceptable_drift_ - average_frame_duration_; |
| 220 for (; frames_to_expire < frame_queue_.size() - 1; ++frames_to_expire) { |
| 221 if (frame_queue_[frames_to_expire].wall_clock_time >= minimum_frame_time) |
| 222 break; |
| 223 } |
| 224 |
| 225 if (!frames_to_expire) |
| 226 return 0; |
| 227 |
| 228 frame_queue_.erase(frame_queue_.begin(), |
| 229 frame_queue_.begin() + frames_to_expire); |
| 230 |
| 231 last_frame_index_ = last_frame_index_ > frames_to_expire |
| 232 ? last_frame_index_ - frames_to_expire |
| 233 : 0; |
| 234 return frames_to_expire; |
| 235 } |
| 236 |
| 237 void VideoRendererAlgorithm::OnLastFrameDropped() { |
| 238 DCHECK(have_rendered_frames_); |
| 239 DCHECK(!frame_queue_.empty()); |
| 240 // If frames were expired by RemoveExpiredFrames() this count may be zero when |
| 241 // the OnLastFrameDropped() call comes in. |
| 242 if (!frame_queue_[last_frame_index_].render_count) |
| 243 return; |
| 244 |
| 245 ++frame_queue_[last_frame_index_].drop_count; |
| 246 DCHECK_LE(frame_queue_[last_frame_index_].drop_count, |
| 247 frame_queue_[last_frame_index_].render_count); |
| 248 } |
| 249 |
| 250 void VideoRendererAlgorithm::Reset() { |
| 251 last_frame_index_ = 0; |
| 252 have_rendered_frames_ = last_render_had_glitch_ = false; |
| 253 last_deadline_max_ = base::TimeTicks(); |
| 254 average_frame_duration_ = render_interval_ = base::TimeDelta(); |
| 255 frame_queue_.clear(); |
| 256 cadence_estimator_.Reset(); |
| 257 frame_duration_calculator_.Reset(); |
| 258 |
| 259 // Default to ATSC IS/191 recommendations for maximum acceptable drift before |
| 260 // we have enough frames to base the maximum on frame duration. |
| 261 max_acceptable_drift_ = base::TimeDelta::FromMilliseconds(15); |
| 262 } |
| 263 |
| 264 size_t VideoRendererAlgorithm::EffectiveFramesQueued() const { |
| 265 if (frame_queue_.empty() || average_frame_duration_ == base::TimeDelta() || |
| 266 last_deadline_max_.is_null()) { |
| 267 return frame_queue_.size(); |
| 268 } |
| 269 |
| 270 // If we don't have cadence, subtract off any frames which are before |
| 271 // the last rendered frame or are past their expected rendering time. |
| 272 if (!cadence_estimator_.has_cadence()) { |
| 273 size_t expired_frames = last_frame_index_; |
| 274 DCHECK_LT(last_frame_index_, frame_queue_.size()); |
| 275 for (; expired_frames < frame_queue_.size(); ++expired_frames) { |
| 276 if (frame_queue_[expired_frames].wall_clock_time.is_null() || |
| 277 EndTimeForFrame(expired_frames) > last_deadline_max_) { |
| 278 break; |
| 279 } |
| 280 } |
| 281 return frame_queue_.size() - expired_frames; |
| 282 } |
| 283 |
| 284 // Find the first usable frame to start counting from. |
| 285 const int start_index = FindBestFrameByCadenceInternal(nullptr); |
| 286 if (start_index < 0) |
| 287 return 0; |
| 288 |
| 289 size_t renderable_frame_count = 0; |
| 290 for (size_t i = start_index; i < frame_queue_.size(); ++i) { |
| 291 if (frame_queue_[i].render_count < frame_queue_[i].ideal_render_count) |
| 292 ++renderable_frame_count; |
| 293 } |
| 294 |
| 295 return renderable_frame_count; |
| 296 } |
| 297 |
| 298 void VideoRendererAlgorithm::EnqueueFrame( |
| 299 const scoped_refptr<VideoFrame>& frame) { |
| 300 DCHECK(frame); |
| 301 DCHECK(!frame->end_of_stream()); |
| 302 |
| 303 ReadyFrame ready_frame(frame); |
| 304 auto it = frame_queue_.empty() ? frame_queue_.end() |
| 305 : std::lower_bound(frame_queue_.begin(), |
| 306 frame_queue_.end(), frame); |
| 307 DCHECK_GE(it - frame_queue_.begin(), 0); |
| 308 |
| 309 // If a frame was inserted before the first frame, update the index. On the |
| 310 // next call to Render() it will be dropped. |
| 311 if (static_cast<size_t>(it - frame_queue_.begin()) <= last_frame_index_ && |
| 312 have_rendered_frames_) { |
| 313 ++last_frame_index_; |
| 314 } |
| 315 |
| 316 // The vast majority of cases should always append to the back, but in rare |
| 317 // circumstance we get out of order timestamps, http://crbug.com/386551. |
| 318 it = frame_queue_.insert(it, ready_frame); |
| 319 |
| 320 // Project the current cadence calculations to include the new frame. These |
| 321 // may not be accurate until the next Render() call. These updates are done |
| 322 // to ensure EffectiveFramesQueued() returns a semi-reliable result. |
| 323 if (cadence_estimator_.has_cadence()) |
| 324 UpdateCadenceForFrames(); |
| 325 |
| 326 #ifndef NDEBUG |
| 327 // Verify sorted order in debug mode. |
| 328 for (size_t i = 0; i < frame_queue_.size() - 1; ++i) { |
| 329 DCHECK(frame_queue_[i].frame->timestamp() <= |
| 330 frame_queue_[i + 1].frame->timestamp()); |
| 331 } |
| 332 #endif |
| 333 } |
| 334 |
| 335 void VideoRendererAlgorithm::AccountForMissedIntervals( |
| 336 base::TimeTicks deadline_min, |
| 337 base::TimeTicks deadline_max) { |
| 338 if (last_deadline_max_.is_null() || deadline_min <= last_deadline_max_ || |
| 339 !have_rendered_frames_) { |
| 340 return; |
| 341 } |
| 342 |
| 343 DCHECK_GT(render_interval_, base::TimeDelta()); |
| 344 const int64 render_cycle_count = |
| 345 (deadline_min - last_deadline_max_) / render_interval_; |
| 346 |
| 347 // In the ideal case this value will be zero. |
| 348 if (!render_cycle_count) |
| 349 return; |
| 350 |
| 351 DVLOG(2) << "Missed " << render_cycle_count << " Render() intervals."; |
| 352 |
| 353 // Only update render count if the frame was rendered at all; it may not have |
| 354 // been if the frame is at the head because we haven't rendered anything yet |
| 355 // or because previous frames were removed via RemoveExpiredFrames(). |
| 356 ReadyFrame& ready_frame = frame_queue_[last_frame_index_]; |
| 357 if (!ready_frame.render_count) |
| 358 return; |
| 359 |
| 360 // If the frame was never really rendered since it was dropped each attempt, |
| 361 // we need to increase the drop count as well to match the new render count. |
| 362 // Otherwise we won't properly count the frame as dropped when it's discarded. |
| 363 // We always update the render count so FindBestFrameByCadenceInternal() can |
| 364 // properly account for potentially over-rendered frames. |
| 365 if (ready_frame.render_count == ready_frame.drop_count) |
| 366 ready_frame.drop_count += render_cycle_count; |
| 367 ready_frame.render_count += render_cycle_count; |
| 368 } |
| 369 |
| 370 bool VideoRendererAlgorithm::UpdateFrameStatistics() { |
| 371 // Figure out all current ready frame times at once so we minimize the drift |
| 372 // relative to real time as the code below executes. |
| 373 for (size_t i = 0; i < frame_queue_.size(); ++i) { |
| 374 ReadyFrame& frame = frame_queue_[i]; |
| 375 const bool new_frame = frame.wall_clock_time.is_null(); |
| 376 frame.wall_clock_time = time_converter_cb_.Run(frame.frame->timestamp()); |
| 377 |
| 378 // If time stops or never started, exit immediately. |
| 379 if (frame.wall_clock_time.is_null()) |
| 380 return false; |
| 381 |
| 382 // TODO(dalecurtis): An unlucky tick of a playback rate change could cause |
| 383 // this to skew so much that time goes backwards between calls. Fix this by |
| 384 // either converting all timestamps at once or with some retry logic. |
| 385 if (i > 0) { |
| 386 const base::TimeDelta delta = |
| 387 frame.wall_clock_time - frame_queue_[i - 1].wall_clock_time; |
| 388 CHECK_GT(delta, base::TimeDelta()); |
| 389 if (new_frame) |
| 390 frame_duration_calculator_.AddSample(delta); |
| 391 } |
| 392 } |
| 393 |
| 394 // Do we have enough frames to compute statistics? |
| 395 const bool have_frame_duration = average_frame_duration_ != base::TimeDelta(); |
| 396 if (frame_queue_.size() < 2 && !have_frame_duration) |
| 397 return false; |
| 398 |
| 399 // Compute |average_frame_duration_|, a moving average of the last few frames; |
| 400 // see kMovingAverageSamples for the exact number. |
| 401 average_frame_duration_ = frame_duration_calculator_.Average(); |
| 402 |
| 403 // ITU-R BR.265 recommends a maximum acceptable drift of +/- half of the frame |
| 404 // duration; there are other asymmetric, more lenient measures, that we're |
| 405 // forgoing in favor of simplicity. |
| 406 // |
| 407 // We'll always allow at least 8.33ms of drift since literature suggests it's |
| 408 // well below the floor of detection. |
| 409 max_acceptable_drift_ = std::max(average_frame_duration_ / 2, |
| 410 base::TimeDelta::FromSecondsD(1.0 / 120)); |
| 411 |
| 412 // If we were called via RemoveExpiredFrames() and Render() was never called, |
| 413 // we may not have a render interval yet. |
| 414 if (render_interval_ == base::TimeDelta()) |
| 415 return true; |
| 416 |
| 417 const bool cadence_changed = cadence_estimator_.UpdateCadenceEstimate( |
| 418 render_interval_, average_frame_duration_, max_acceptable_drift_); |
| 419 |
| 420 // No need to update cadence if there's been no change; cadence will be set |
| 421 // as frames are added to the queue. |
| 422 if (!cadence_changed) |
| 423 return true; |
| 424 |
| 425 UpdateCadenceForFrames(); |
| 426 |
| 427 // Thus far there appears to be no need for special 3:2 considerations, the |
| 428 // smoothness scores seem to naturally fit that pattern based on maximizing |
| 429 // frame coverage. |
| 430 return true; |
| 431 } |
| 432 |
| 433 void VideoRendererAlgorithm::UpdateCadenceForFrames() { |
| 434 for (size_t i = last_frame_index_; i < frame_queue_.size(); ++i) { |
| 435 // It's always okay to adjust the ideal render count, since the cadence |
| 436 // selection method will still count its current render count towards |
| 437 // cadence selection. |
| 438 frame_queue_[i].ideal_render_count = |
| 439 cadence_estimator_.has_cadence() |
| 440 ? cadence_estimator_.GetCadenceForFrame(i - last_frame_index_) |
| 441 : 0; |
| 442 } |
| 443 } |
| 444 |
| 445 int VideoRendererAlgorithm::FindBestFrameByCadence() { |
| 446 DCHECK(!frame_queue_.empty()); |
| 447 if (!cadence_estimator_.has_cadence()) |
| 448 return -1; |
| 449 |
| 450 int new_ideal_render_count = 0; |
| 451 const int best_frame = |
| 452 FindBestFrameByCadenceInternal(&new_ideal_render_count); |
| 453 if (best_frame < 0) |
| 454 return -1; |
| 455 |
| 456 DCHECK_GT(new_ideal_render_count, 0); |
| 457 frame_queue_[best_frame].ideal_render_count = new_ideal_render_count; |
| 458 return best_frame; |
| 459 } |
| 460 |
| 461 int VideoRendererAlgorithm::FindBestFrameByCadenceInternal( |
| 462 int* adjusted_ideal_render_count) const { |
| 463 DCHECK(!frame_queue_.empty()); |
| 464 DCHECK(cadence_estimator_.has_cadence()); |
| 465 const ReadyFrame& current_frame = frame_queue_[last_frame_index_]; |
| 466 |
| 467 // If the current frame is below cadence, we should prefer it. |
| 468 if (current_frame.render_count < current_frame.ideal_render_count) { |
| 469 if (adjusted_ideal_render_count) |
| 470 *adjusted_ideal_render_count = current_frame.ideal_render_count; |
| 471 return last_frame_index_; |
| 472 } |
| 473 |
| 474 // For over-rendered frames we need to ensure we skip frames and subtract |
| 475 // each skipped frame's ideal cadence from the over-render count until we |
| 476 // find a frame which still has a positive ideal render count. |
| 477 int render_count_overage = std::max( |
| 478 0, current_frame.render_count - current_frame.ideal_render_count); |
| 479 |
| 480 // If the current frame is on cadence or over cadence, find the next frame |
| 481 // with a positive ideal render count. |
| 482 for (size_t i = last_frame_index_ + 1; i < frame_queue_.size(); ++i) { |
| 483 const ReadyFrame& frame = frame_queue_[i]; |
| 484 if (frame.ideal_render_count > render_count_overage) { |
| 485 if (adjusted_ideal_render_count) { |
| 486 *adjusted_ideal_render_count = |
| 487 frame.ideal_render_count - render_count_overage; |
| 488 } |
| 489 return i; |
| 490 } else { |
| 491 // The ideal render count should always be zero or smaller than the |
| 492 // over-render count. |
| 493 render_count_overage -= frame.ideal_render_count; |
| 494 DCHECK_GE(render_count_overage, 0); |
| 495 } |
| 496 } |
| 497 |
| 498 // We don't have enough frames to find a better once by cadence. |
| 499 return -1; |
| 500 } |
| 501 |
| 502 int VideoRendererAlgorithm::FindBestFrameByCoverage( |
| 503 base::TimeTicks deadline_min, |
| 504 base::TimeTicks deadline_max, |
| 505 int* second_best) const { |
| 506 DCHECK(!frame_queue_.empty()); |
| 507 |
| 508 // Find the frame which covers the most of the interval [deadline_min, |
| 509 // deadline_max]. Frames outside of the interval are considered to have no |
| 510 // coverage, while those which completely overlap the interval have complete |
| 511 // coverage. |
| 512 int best_frame_by_coverage = -1; |
| 513 base::TimeDelta best_coverage; |
| 514 std::vector<base::TimeDelta> coverage(frame_queue_.size(), base::TimeDelta()); |
| 515 for (size_t i = last_frame_index_; i < frame_queue_.size(); ++i) { |
| 516 // Frames which start after the deadline interval have zero coverage. |
| 517 if (frame_queue_[i].wall_clock_time > deadline_max) |
| 518 break; |
| 519 |
| 520 // Clamp frame end times to a maximum of |deadline_max|. |
| 521 const base::TimeTicks frame_end_time = |
| 522 std::min(deadline_max, EndTimeForFrame(i)); |
| 523 |
| 524 // Frames entirely before the deadline interval have zero coverage. |
| 525 if (frame_end_time < deadline_min) |
| 526 continue; |
| 527 |
| 528 // If we're here, the current frame overlaps the deadline in some way; so |
| 529 // compute the duration of the interval which is covered. |
| 530 const base::TimeDelta duration = |
| 531 frame_end_time - |
| 532 std::max(deadline_min, frame_queue_[i].wall_clock_time); |
| 533 |
| 534 coverage[i] = duration; |
| 535 if (coverage[i] > best_coverage) { |
| 536 best_frame_by_coverage = i; |
| 537 best_coverage = coverage[i]; |
| 538 } |
| 539 } |
| 540 |
| 541 // Find the second best frame by coverage; done by zeroing the coverage for |
| 542 // the previous best and recomputing the maximum. |
| 543 *second_best = -1; |
| 544 if (best_frame_by_coverage >= 0) { |
| 545 coverage[best_frame_by_coverage] = base::TimeDelta(); |
| 546 auto it = std::max_element(coverage.begin(), coverage.end()); |
| 547 if (*it > base::TimeDelta()) |
| 548 *second_best = it - coverage.begin(); |
| 549 } |
| 550 |
| 551 // If two frames have coverage within half a millisecond, prefer the earliest |
| 552 // frame as having the best coverage. Value chosen via experimentation to |
| 553 // ensure proper coverage calculation for 24fps in 60Hz where +/- 100us of |
| 554 // jitter is present within the |render_interval_|. At 60Hz this works out to |
| 555 // an allowed jitter of 3%. |
| 556 const base::TimeDelta kAllowableJitter = |
| 557 base::TimeDelta::FromMicroseconds(500); |
| 558 if (*second_best >= 0 && best_frame_by_coverage > *second_best && |
| 559 (best_coverage - coverage[*second_best]).magnitude() <= |
| 560 kAllowableJitter) { |
| 561 std::swap(best_frame_by_coverage, *second_best); |
| 562 } |
| 563 |
| 564 // TODO(dalecurtis): We may want to make a better decision about what to do |
| 565 // when multiple frames have equivalent coverage over an interval. Jitter in |
| 566 // the render interval may result in irregular frame selection which may be |
| 567 // visible to a viewer. |
| 568 // |
| 569 // 23.974fps and 24fps in 60Hz are the most common susceptible rates, so |
| 570 // extensive tests have been added to ensure these cases work properly. |
| 571 |
| 572 return best_frame_by_coverage; |
| 573 } |
| 574 |
| 575 int VideoRendererAlgorithm::FindBestFrameByDrift( |
| 576 base::TimeTicks deadline_min, |
| 577 base::TimeDelta* selected_frame_drift) const { |
| 578 DCHECK(!frame_queue_.empty()); |
| 579 |
| 580 int best_frame_by_drift = -1; |
| 581 *selected_frame_drift = base::TimeDelta::Max(); |
| 582 |
| 583 for (size_t i = last_frame_index_; i < frame_queue_.size(); ++i) { |
| 584 const base::TimeDelta drift = |
| 585 CalculateAbsoluteDriftForFrame(deadline_min, i); |
| 586 // We use <= here to prefer the latest frame with minimum drift. |
| 587 if (drift <= *selected_frame_drift) { |
| 588 *selected_frame_drift = drift; |
| 589 best_frame_by_drift = i; |
| 590 } |
| 591 } |
| 592 |
| 593 return best_frame_by_drift; |
| 594 } |
| 595 |
| 596 base::TimeDelta VideoRendererAlgorithm::CalculateAbsoluteDriftForFrame( |
| 597 base::TimeTicks deadline_min, |
| 598 int frame_index) const { |
| 599 // If the frame lies before the deadline, compute the delta against the end |
| 600 // of the frame's duration. |
| 601 const base::TimeTicks frame_end_time = EndTimeForFrame(frame_index); |
| 602 if (frame_end_time < deadline_min) |
| 603 return deadline_min - frame_end_time; |
| 604 |
| 605 // If the frame lies after the deadline, compute the delta against the frame's |
| 606 // wall clock time. |
| 607 const ReadyFrame& frame = frame_queue_[frame_index]; |
| 608 if (frame.wall_clock_time > deadline_min) |
| 609 return frame.wall_clock_time - deadline_min; |
| 610 |
| 611 // Drift is zero for frames which overlap the deadline interval. |
| 612 DCHECK_GE(deadline_min, frame.wall_clock_time); |
| 613 DCHECK_GE(frame_end_time, deadline_min); |
| 614 return base::TimeDelta(); |
| 615 } |
| 616 |
| 617 base::TimeTicks VideoRendererAlgorithm::EndTimeForFrame( |
| 618 size_t frame_index) const { |
| 619 DCHECK_LT(frame_index, frame_queue_.size()); |
| 620 DCHECK_GT(average_frame_duration_, base::TimeDelta()); |
| 621 return frame_index + 1 < frame_queue_.size() |
| 622 ? frame_queue_[frame_index + 1].wall_clock_time |
| 623 : frame_queue_[frame_index].wall_clock_time + |
| 624 average_frame_duration_; |
| 625 } |
| 626 |
| 627 } // namespace media |
OLD | NEW |