Connect the new video rendering path to the compositor.

media/renderers/video_renderer_impl.cc

Index: media/renderers/video_renderer_impl.cc
diff --git a/media/renderers/video_renderer_impl.cc b/media/renderers/video_renderer_impl.cc
index 3e0698f14266106940b6c6047f240b2698995b46..79753cd800707bab68e22a9ad595031e6d5854c7 100644
--- a/media/renderers/video_renderer_impl.cc
+++ b/media/renderers/video_renderer_impl.cc
@@ -7,6 +7,7 @@
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
+#include "base/command_line.h"
 #include "base/location.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/platform_thread.h"
@@ -15,11 +16,16 @@
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/buffers.h"
 #include "media/base/limits.h"
+#include "media/base/media_switches.h"
 #include "media/base/pipeline.h"
 #include "media/base/video_frame.h"
 
 namespace media {
 
+// Wait a quarter of a second for Render() callbacks before starting background
+// rendering to keep the decode pump moving.
+static const int kBackgroundRenderingTimeoutMs = 250;

[xhwang, 2015/04/29 06:51:54]

nit: s/static//

[DaleCurtis, 2015/04/30 03:49:37]

Done.

+
 VideoRendererImpl::VideoRendererImpl(
     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
     VideoRendererSink* sink,
@@ -27,6 +33,9 @@ VideoRendererImpl::VideoRendererImpl(
     bool drop_frames,
     const scoped_refptr<MediaLog>& media_log)
     : task_runner_(task_runner),
+      use_new_video_renderering_path_(
+          base::CommandLine::ForCurrentProcess()->HasSwitch(
+              switches::kEnableNewVideoRenderer)),
       sink_(sink),
       video_frame_stream_(
           new VideoFrameStream(task_runner, decoders.Pass(), media_log)),
@@ -43,13 +52,19 @@ VideoRendererImpl::VideoRendererImpl(
       frames_dropped_(0),
       is_shutting_down_(false),
       tick_clock_(new base::DefaultTickClock()),
+      is_background_rendering_(false),
+      should_use_background_renderering_(true),
+      time_progressing_(false),
+      render_first_frame_and_stop_(false),
+      background_rendering_timeout_(
+          base::TimeDelta::FromMilliseconds(kBackgroundRenderingTimeoutMs)),
       weak_factory_(this) {
 }
 
 VideoRendererImpl::~VideoRendererImpl() {
   DCHECK(task_runner_->BelongsToCurrentThread());
 
-  {
+  if (!use_new_video_renderering_path_) {
     base::AutoLock auto_lock(lock_);
     is_shutting_down_ = true;
     frame_available_.Signal();
@@ -63,6 +78,11 @@ VideoRendererImpl::~VideoRendererImpl() {
 
   if (!flush_cb_.is_null())
     base::ResetAndReturn(&flush_cb_).Run();
+
+  if (use_new_video_renderering_path_ &&
+      (time_progressing_ || render_first_frame_and_stop_)) {
+    StopSink();
+  }
 }
 
 void VideoRendererImpl::Flush(const base::Closure& callback) {
@@ -83,6 +103,9 @@ void VideoRendererImpl::Flush(const base::Closure& callback) {
   received_end_of_stream_ = false;
   rendered_end_of_stream_ = false;
 
+  if (use_new_video_renderering_path_)
+    algorithm_->Reset();
+
   video_frame_stream_->Reset(
       base::Bind(&VideoRendererImpl::OnVideoFrameStreamResetDone,
                  weak_factory_.GetWeakPtr()));
@@ -137,6 +160,9 @@ void VideoRendererImpl::Initialize(
   error_cb_ = error_cb;
   wall_clock_time_cb_ = wall_clock_time_cb;
   state_ = kInitializing;
+  render_first_frame_and_stop_ = false;
+  is_background_rendering_ = false;
+  time_progressing_ = false;

[xhwang, 2015/04/29 06:51:55]

Can we DCHECK these three?

[DaleCurtis, 2015/04/30 03:49:37]

Done.

 
   video_frame_stream_->Initialize(
       stream, base::Bind(&VideoRendererImpl::OnVideoFrameStreamInitialized,
@@ -147,14 +173,65 @@ void VideoRendererImpl::Initialize(
 scoped_refptr<VideoFrame> VideoRendererImpl::Render(
     base::TimeTicks deadline_min,
     base::TimeTicks deadline_max) {
-  // TODO(dalecurtis): Hook this up to the new VideoRendererAlgorithm.
-  NOTIMPLEMENTED();
-  return nullptr;
+  base::AutoLock auto_lock(lock_);
+  DCHECK(use_new_video_renderering_path_);
+
+  size_t frames_dropped = 0;
+  scoped_refptr<VideoFrame> result =
+      algorithm_->Render(deadline_min, deadline_max, &frames_dropped);
+
+  // Due to how the |algorithm_| holds frames, this should never be null if
+  // we've had a proper startup sequence.
+  DCHECK(result);
+
+  // See if it's time to fire the ended callback.
+  const size_t effective_frames = MaybeFireEndedCallback();
+
+  // Declare HAVE_NOTHING if we have no more effective frames.
+  if (!effective_frames && !rendered_end_of_stream_ &&
+      buffering_state_ == BUFFERING_HAVE_ENOUGH) {
+    buffering_state_ = BUFFERING_HAVE_NOTHING;
+    task_runner_->PostTask(
+        FROM_HERE, base::Bind(buffering_state_cb_, BUFFERING_HAVE_NOTHING));
+  }
+
+  // As we resume from background rendering, don't count the initial batch of
+  // dropped frames since they are likely just dropped due to being too old.
+  if (!is_background_rendering_) {
+    frames_dropped_ += frames_dropped;
+    UpdateStatsAndWait_Locked(base::TimeDelta());
+  }
+  is_background_rendering_ = false;
+
+  // After painting the first frame, if playback hasn't started, we request that
+  // the sink be stopped.  OnTimeStateChanged() will clear this flag if time
+  // starts before we get here and MaybeStopSinkAfterFirstPaint() will ignore
+  // this request if time starts before the call executes.
+  if (render_first_frame_and_stop_) {
+    task_runner_->PostTask(
+        FROM_HERE, base::Bind(&VideoRendererImpl::MaybeStopSinkAfterFirstPaint,
+                              weak_factory_.GetWeakPtr()));
+  }
+
+  // Always post this task, it will acquire new frames if necessary, reset the
+  // background rendering timer, and more.
+  task_runner_->PostTask(FROM_HERE, base::Bind(&VideoRendererImpl::AttemptRead,
+                                               weak_factory_.GetWeakPtr()));
+
+  return result;
 }
 
 void VideoRendererImpl::OnFrameDropped() {
-  // TODO(dalecurtis): Hook this up to the new VideoRendererAlgorithm.
-  NOTIMPLEMENTED();
+  base::AutoLock auto_lock(lock_);
+  DCHECK(use_new_video_renderering_path_);
+  algorithm_->OnLastFrameDropped();
+}
+
+void VideoRendererImpl::SetBackgroundRenderingForTesting(
+    bool enabled,
+    base::TimeDelta timeout) {
+  should_use_background_renderering_ = enabled;
+  background_rendering_timeout_ = timeout;
 }
 
 void VideoRendererImpl::CreateVideoThread() {
@@ -186,13 +263,20 @@ void VideoRendererImpl::OnVideoFrameStreamInitialized(bool success) {
   // have not populated any buffers yet.
   state_ = kFlushed;
 
-  CreateVideoThread();
+  if (use_new_video_renderering_path_) {
+    algorithm_.reset(new VideoRendererAlgorithm(wall_clock_time_cb_));
+    if (!drop_frames_)
+      algorithm_->disable_frame_dropping();
+  } else {
+    CreateVideoThread();
+  }
 
   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
 }
 
 // PlatformThread::Delegate implementation.
 void VideoRendererImpl::ThreadMain() {
+  DCHECK(!use_new_video_renderering_path_);
   base::PlatformThread::SetName("CrVideoRenderer");
 
   // The number of milliseconds to idle when we do not have anything to do.
@@ -290,10 +374,28 @@ void VideoRendererImpl::SetTickClockForTesting(
 }
 
 void VideoRendererImpl::OnTimeStateChanged(bool time_progressing) {
-  // TODO(dalecurtis): Wire up to the VideoRendererSink once it's implemented.
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  const bool was_progressing = time_progressing_;
+  time_progressing_ = time_progressing;
+
+  // WARNING: Do not attempt to use |lock_| here as this may be a reentrant call
+  // in response to callbacks firing above.
+
+  if (!use_new_video_renderering_path_ ||
+      was_progressing == time_progressing_ || render_first_frame_and_stop_) {
+    return;
+  }
+
+  if (time_progressing_) {
+    StartSink();
+    return;
+  }
+
+  StopSink();
 }
 
 void VideoRendererImpl::PaintNextReadyFrame_Locked() {
+  DCHECK(!use_new_video_renderering_path_);
   lock_.AssertAcquired();
 
   scoped_refptr<VideoFrame> next_frame = ready_frames_.front();
@@ -310,6 +412,7 @@ void VideoRendererImpl::PaintNextReadyFrame_Locked() {
 }
 
 void VideoRendererImpl::DropNextReadyFrame_Locked() {
+  DCHECK(!use_new_video_renderering_path_);
   TRACE_EVENT0("media", "VideoRendererImpl:frameDropped");
 
   lock_.AssertAcquired();
@@ -329,64 +432,101 @@ void VideoRendererImpl::DropNextReadyFrame_Locked() {
 void VideoRendererImpl::FrameReady(VideoFrameStream::Status status,
                                    const scoped_refptr<VideoFrame>& frame) {
   DCHECK(task_runner_->BelongsToCurrentThread());
-  base::AutoLock auto_lock(lock_);
-  DCHECK_NE(state_, kUninitialized);
-  DCHECK_NE(state_, kFlushed);
-
-  CHECK(pending_read_);
-  pending_read_ = false;
-
-  if (status == VideoFrameStream::DECODE_ERROR ||
-      status == VideoFrameStream::DECRYPT_ERROR) {
-    DCHECK(!frame.get());
-    PipelineStatus error = PIPELINE_ERROR_DECODE;
-    if (status == VideoFrameStream::DECRYPT_ERROR)
-      error = PIPELINE_ERROR_DECRYPT;
-    task_runner_->PostTask(FROM_HERE, base::Bind(error_cb_, error));
-    return;
-  }
+  bool start_sink = false;
+  {
+    base::AutoLock auto_lock(lock_);
+    DCHECK_NE(state_, kUninitialized);
+    DCHECK_NE(state_, kFlushed);
+
+    CHECK(pending_read_);
+    pending_read_ = false;
+
+    if (status == VideoFrameStream::DECODE_ERROR ||
+        status == VideoFrameStream::DECRYPT_ERROR) {
+      DCHECK(!frame.get());
+      PipelineStatus error = PIPELINE_ERROR_DECODE;
+      if (status == VideoFrameStream::DECRYPT_ERROR)
+        error = PIPELINE_ERROR_DECRYPT;
+      task_runner_->PostTask(FROM_HERE, base::Bind(error_cb_, error));
+      return;
+    }
 
-  // Already-queued VideoFrameStream ReadCB's can fire after various state
-  // transitions have happened; in that case just drop those frames immediately.
-  if (state_ == kFlushing)
-    return;
+    // Already-queued VideoFrameStream ReadCB's can fire after various state
+    // transitions have happened; in that case just drop those frames
+    // immediately.
+    if (state_ == kFlushing)
+      return;
 
-  DCHECK_EQ(state_, kPlaying);
+    DCHECK_EQ(state_, kPlaying);
 
-  // Can happen when demuxers are preparing for a new Seek().
-  if (!frame.get()) {
-    DCHECK_EQ(status, VideoFrameStream::DEMUXER_READ_ABORTED);
-    return;
-  }
+    // Can happen when demuxers are preparing for a new Seek().
+    if (!frame.get()) {
+      DCHECK_EQ(status, VideoFrameStream::DEMUXER_READ_ABORTED);
+      return;
+    }
 
-  if (frame->end_of_stream()) {
-    DCHECK(!received_end_of_stream_);
-    received_end_of_stream_ = true;
-  } else {
-    // Maintain the latest frame decoded so the correct frame is displayed after
-    // prerolling has completed.
-    if (frame->timestamp() <= start_timestamp_)
-      ready_frames_.clear();
-    AddReadyFrame_Locked(frame);
-  }
+    if (frame->end_of_stream()) {
+      DCHECK(!received_end_of_stream_);
+      received_end_of_stream_ = true;
+
+      // See if we can fire EOS immediately instead of waiting for Render() or
+      // BackgroundRender() to tick.
+      if (use_new_video_renderering_path_ && time_progressing_)
+        MaybeFireEndedCallback();
+    } else {
+      // Maintain the latest frame decoded so the correct frame is displayed
+      // after prerolling has completed.
+      if (frame->timestamp() <= start_timestamp_) {
+        if (use_new_video_renderering_path_)
+          algorithm_->Reset();
+        ready_frames_.clear();
+      }
+      AddReadyFrame_Locked(frame);
+    }
 
-  // Signal buffering state if we've met our conditions for having enough data.
-  if (buffering_state_ != BUFFERING_HAVE_ENOUGH && HaveEnoughData_Locked())
-    TransitionToHaveEnough_Locked();
+    // Signal buffering state if we've met our conditions for having enough
+    // data.
+    if (buffering_state_ != BUFFERING_HAVE_ENOUGH && HaveEnoughData_Locked()) {
+      TransitionToHaveEnough_Locked();
+      start_sink = true;
+      render_first_frame_and_stop_ = true;
+    }
 
-  // Always request more decoded video if we have capacity. This serves two
-  // purposes:
-  //   1) Prerolling while paused
-  //   2) Keeps decoding going if video rendering thread starts falling behind
-  AttemptRead_Locked();
+    // BackgroundRender may not be ticking fast enough by itself to remove
+    // expired frames, so give it a boost here by ensuring we don't exit the
+    // decoding cycle too early.
+    if (is_background_rendering_) {
+      DCHECK(use_new_video_renderering_path_);
+      BackgroundRender_Locked();
+    }
+
+    // Always request more decoded video if we have capacity. This serves two
+    // purposes:
+    //   1) Prerolling while paused
+    //   2) Keeps decoding going if video rendering thread starts falling behind
+    AttemptRead_Locked();
+  }
+
+  // If time is progressing, the sink has already been started; this may be true
+  // if we have previously underflowed, yet weren't stopped because of audio.
+  if (use_new_video_renderering_path_ && !time_progressing_ && start_sink)
+    StartSink();
 }
 
 bool VideoRendererImpl::HaveEnoughData_Locked() {
   DCHECK_EQ(state_, kPlaying);
-  return received_end_of_stream_ ||
-      !video_frame_stream_->CanReadWithoutStalling() ||
-      ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames) ||
-      (low_delay_ && ready_frames_.size() > 0);
+
+  if (received_end_of_stream_ || !video_frame_stream_->CanReadWithoutStalling())
+    return true;
+
+  if (HaveReachedBufferingCap())
+    return true;
+
+  if (!low_delay_)
+    return false;
+
+  return ready_frames_.size() > 0 ||
+         (use_new_video_renderering_path_ && algorithm_->frames_queued() > 0);
 }
 
 void VideoRendererImpl::TransitionToHaveEnough_Locked() {
@@ -394,6 +534,7 @@ void VideoRendererImpl::TransitionToHaveEnough_Locked() {
   DCHECK_EQ(buffering_state_, BUFFERING_HAVE_NOTHING);
 
   if (!ready_frames_.empty()) {
+    DCHECK(!use_new_video_renderering_path_);
     // Because the clock might remain paused in for an undetermined amount
     // of time (e.g., seeking while paused), paint the first frame.
     PaintNextReadyFrame_Locked();
@@ -409,6 +550,11 @@ void VideoRendererImpl::AddReadyFrame_Locked(
   lock_.AssertAcquired();
   DCHECK(!frame->end_of_stream());
 
+  if (use_new_video_renderering_path_) {
+    algorithm_->EnqueueFrame(frame);
+    return;
+  }
+
   ready_frames_.push_back(frame);
   DCHECK_LE(ready_frames_.size(),
             static_cast<size_t>(limits::kMaxVideoFrames));
@@ -427,10 +573,14 @@ void VideoRendererImpl::AttemptRead_Locked() {
   DCHECK(task_runner_->BelongsToCurrentThread());
   lock_.AssertAcquired();
 
-  if (pending_read_ || received_end_of_stream_ ||
-      ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) {
+  if (pending_read_ || received_end_of_stream_)
+    return;
+
+  if (use_new_video_renderering_path_ && time_progressing_)
+    RestartBackgroundRenderTimer();

[xhwang, 2015/04/29 06:51:54]

It's a bit unclear to me when we should call this. Let's chat on this tomorrow
:)

[DaleCurtis, 2015/04/30 03:49:37]

Per offline discussion replaced with a repeating timer which checks a last known
render time.

+
+  if (HaveReachedBufferingCap())
     return;
-  }
 
   switch (state_) {
     case kPlaying:
@@ -458,6 +608,7 @@ void VideoRendererImpl::OnVideoFrameStreamResetDone() {
 
   state_ = kFlushed;
   latest_possible_paint_time_ = last_media_time_ = base::TimeTicks();
+
   base::ResetAndReturn(&flush_cb_).Run();
 }
 
@@ -465,9 +616,9 @@ void VideoRendererImpl::UpdateStatsAndWait_Locked(
     base::TimeDelta wait_duration) {
   lock_.AssertAcquired();
   DCHECK_GE(frames_decoded_, 0);
-  DCHECK_LE(frames_dropped_, frames_decoded_);
+  DCHECK_GE(frames_dropped_, 0);
 
-  if (frames_decoded_) {
+  if (frames_decoded_ || frames_dropped_) {
     PipelineStatistics statistics;
     statistics.video_frames_decoded = frames_decoded_;
     statistics.video_frames_dropped = frames_dropped_;
@@ -477,7 +628,122 @@ void VideoRendererImpl::UpdateStatsAndWait_Locked(
     frames_dropped_ = 0;
   }
 
-  frame_available_.TimedWait(wait_duration);
+  if (wait_duration > base::TimeDelta())
+    frame_available_.TimedWait(wait_duration);
+}
+
+void VideoRendererImpl::MaybeStopSinkAfterFirstPaint() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  DCHECK(use_new_video_renderering_path_);
+
+  {
+    base::AutoLock auto_lock(lock_);
+    render_first_frame_and_stop_ = false;
+  }
+
+  if (!time_progressing_)
+    StopSink();
+}
+
+void VideoRendererImpl::RestartBackgroundRenderTimer() {
+  DCHECK(use_new_video_renderering_path_);
+  if (!drop_frames_ || !should_use_background_renderering_)
+    return;
+
+  // This controls the amount of time allowed between Render() callbacks before
+  // we assume they have timed out and it's necessary to start expiring
+  // unrendered frames.
+  background_rendering_timer_.Start(
+      FROM_HERE, background_rendering_timeout_,
+      base::Bind(&VideoRendererImpl::BackgroundRender,
+                 weak_factory_.GetWeakPtr()));
+}
+
+void VideoRendererImpl::BackgroundRender() {
+  base::AutoLock auto_lock(lock_);
+  BackgroundRender_Locked();
+}
+
+void VideoRendererImpl::BackgroundRender_Locked() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  lock_.AssertAcquired();
+
+  // If a Render() call never occurs after starting playback for the first frame
+  // we need to carry out the duties of Render() and stop the sink.  We don't
+  // call MaybeStopSinkAfterFirstPaint() since it may need to mutate |sink_|,
+  // which can't be done under lock.
+  if (render_first_frame_and_stop_) {
+    task_runner_->PostTask(
+        FROM_HERE, base::Bind(&VideoRendererImpl::MaybeStopSinkAfterFirstPaint,
+                              weak_factory_.GetWeakPtr()));
+
+    // MaybeStopSinkAfterFirstPaint isn't going to stop the sink if time is
+    // currently progressing, so only bail out if necessary.
+    if (!time_progressing_) {
+      RestartBackgroundRenderTimer();
+      return;
+    }
+  }
+
+  // First clear as many expired frames as we can.
+  algorithm_->RemoveExpiredFrames(tick_clock_->NowTicks());
+  is_background_rendering_ = true;
+
+  // See if we've run out of frames and need to fire the ended callback.
+  MaybeFireEndedCallback();
+  if (rendered_end_of_stream_)
+    return;
+
+  // Start queuing new frames and scheduled this process again otherwise.
+  AttemptRead_Locked();
+  UpdateStatsAndWait_Locked(base::TimeDelta());
+  RestartBackgroundRenderTimer();
+}
+
+bool VideoRendererImpl::HaveReachedBufferingCap() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  if (use_new_video_renderering_path_) {
+    // When the display rate is less than the frame rate, the effective frames
+    // queued may be much smaller than the actual number of frames queued.  Here
+    // we ensure that frames_queued() doesn't get excessive.
+    return algorithm_->EffectiveFramesQueued() >=
+               static_cast<size_t>(limits::kMaxVideoFrames) ||
+           algorithm_->frames_queued() >=
+               static_cast<size_t>(3 * limits::kMaxVideoFrames);
+  }
+
+  return ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames);
+}
+
+void VideoRendererImpl::StartSink() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  sink_->Start(this);
+  RestartBackgroundRenderTimer();
+}
+
+void VideoRendererImpl::StopSink() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  sink_->Stop();
+  background_rendering_timer_.Stop();
+  is_background_rendering_ = false;
+}
+
+size_t VideoRendererImpl::MaybeFireEndedCallback() {
+  // If there's only one frame in the video or Render() was never called, the
+  // algorithm will have one frame linger indefinitely.  So in cases where the
+  // frame duration is unknown nad we've recieved EOS, fire it once we get down

[xhwang, 2015/04/29 06:51:54]

s/nad/and/

[DaleCurtis, 2015/04/30 03:49:37]

Done.

+  // to a single frame.
+  const size_t effective_frames = algorithm_->EffectiveFramesQueued();
+
+  if ((!effective_frames ||
+       (algorithm_->frames_queued() == 1u &&
+        algorithm_->average_frame_duration() == base::TimeDelta())) &&
+      received_end_of_stream_ && !rendered_end_of_stream_) {
+    rendered_end_of_stream_ = true;
+    task_runner_->PostTask(FROM_HERE, ended_cb_);
+  }
+
+  return effective_frames;
 }
 
 }  // namespace media