Preliminary change for new rtc rendering algorithm

content/renderer/media/webmediaplayer_ms.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webmediaplayer_ms.h"
 
 #include <limits>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "cc/blink/context_provider_web_context.h"
 #include "cc/blink/web_layer_impl.h"
+#include "cc/layers/video_frame_provider_client_impl.h"
 #include "cc/layers/video_layer.h"
 #include "content/public/renderer/media_stream_audio_renderer.h"
 #include "content/public/renderer/media_stream_renderer_factory.h"
 #include "content/public/renderer/render_view.h"
 #include "content/public/renderer/video_frame_provider.h"
 #include "content/renderer/render_frame_impl.h"
 #include "content/renderer/render_thread_impl.h"
 #include "gpu/blink/webgraphicscontext3d_impl.h"
 #include "media/base/media_log.h"
 #include "media/base/video_frame.h"
@@ skipping 61 matching lines @@
   return new_frame;
 }
 
 }  // anonymous namespace
 
 WebMediaPlayerMS::WebMediaPlayerMS(
     blink::WebFrame* frame,
     blink::WebMediaPlayerClient* client,
     base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
     media::MediaLog* media_log,
-    scoped_ptr<MediaStreamRendererFactory> factory)
+    scoped_ptr<MediaStreamRendererFactory> factory,
+    scoped_refptr<base::SingleThreadTaskRunner> compositor_thread)
     : frame_(frame),
       network_state_(WebMediaPlayer::NetworkStateEmpty),
       ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
       buffered_(static_cast<size_t>(0)),
       volume_(1.0f),
       client_(client),
       delegate_(delegate),
       paused_(true),
-      current_frame_used_(false),
-      video_frame_provider_client_(NULL),
       received_first_frame_(false),
-      total_frame_count_(0),
-      dropped_frame_count_(0),
       media_log_(media_log),
-      renderer_factory_(factory.Pass()) {
+      renderer_factory_(factory.Pass()),
+      compositor_(new Compositor(compositor_thread)),
+      compositor_thread_(compositor_thread) {
   DVLOG(1) << "WebMediaPlayerMS::ctor";
   media_log_->AddEvent(
       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
 }
 
 WebMediaPlayerMS::~WebMediaPlayerMS() {
   DVLOG(1) << "WebMediaPlayerMS::dtor";
   DCHECK(thread_checker_.CalledOnValidThread());
 
-  SetVideoFrameProviderClient(NULL);
+  compositor_thread_->DeleteSoon(FROM_HERE, compositor_.release());
+
   GetClient()->setWebLayer(NULL);
 
   if (video_frame_provider_.get())
     video_frame_provider_->Stop();
 
   if (audio_renderer_.get())
     audio_renderer_->Stop();
 
   media_log_->AddEvent(
       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
@@ skipping 48 matching lines @@
 }
 
 void WebMediaPlayerMS::play() {
   DVLOG(1) << "WebMediaPlayerMS::play";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (paused_) {
     if (video_frame_provider_.get())
       video_frame_provider_->Play();
 
+    compositor_thread_->PostTask(
+        FROM_HERE, base::Bind(&WebMediaPlayerMS::Compositor::StartRendering,
+                              base::Unretained(compositor_.get())));
+
     if (audio_renderer_.get())
       audio_renderer_->Play();
 
     if (delegate_.get())
       delegate_->DidPlay(this);
   }
 
   paused_ = false;
 
   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
 }
 
 void WebMediaPlayerMS::pause() {
   DVLOG(1) << "WebMediaPlayerMS::pause";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (video_frame_provider_.get())
     video_frame_provider_->Pause();
 
+  compositor_thread_->PostTask(
+      FROM_HERE, base::Bind(&WebMediaPlayerMS::Compositor::StopRendering,
+                            base::Unretained(compositor_.get())));
+
   if (!paused_) {
     if (audio_renderer_.get())
       audio_renderer_->Pause();
 
     if (delegate_.get())
       delegate_->DidPause(this);
   }
 
   paused_ = true;
 
   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PAUSE));
-
-  if (!current_frame_.get())
-    return;
-
-  // Copy the frame so that rendering can show the last received frame.
-  // The original frame must not be referenced when the player is paused since
-  // there might be a finite number of available buffers. E.g, video that
-  // originates from a video camera.
-  scoped_refptr<media::VideoFrame> new_frame =
-      CopyFrameToYV12(current_frame_, &video_renderer_);
-
-  base::AutoLock auto_lock(current_frame_lock_);
-  current_frame_ = new_frame;
 }
 
 bool WebMediaPlayerMS::supportsSave() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return false;
 }
 
 void WebMediaPlayerMS::seek(double seconds) {
   DCHECK(thread_checker_.CalledOnValidThread());
 }
@@ skipping 37 matching lines @@
 }
 
 bool WebMediaPlayerMS::hasAudio() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return (audio_renderer_.get() != NULL);
 }
 
 blink::WebSize WebMediaPlayerMS::naturalSize() const {
   DCHECK(thread_checker_.CalledOnValidThread());
 
-  gfx::Size size;
-  if (current_frame_.get())
-    size = current_frame_->natural_size();
+  gfx::Size size = compositor_->CurrentFrameSize();
+
   DVLOG(3) << "WebMediaPlayerMS::naturalSize, " << size.ToString();
   return blink::WebSize(size);
 }
 
 bool WebMediaPlayerMS::paused() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return paused_;
 }
 
 bool WebMediaPlayerMS::seeking() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return false;
 }
 
 double WebMediaPlayerMS::duration() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return std::numeric_limits<double>::infinity();
 }
 
 double WebMediaPlayerMS::currentTime() const {
   DCHECK(thread_checker_.CalledOnValidThread());
-  if (current_time_.ToInternalValue() != 0) {
-    return current_time_.InSecondsF();
+  base::TimeDelta current_time = compositor_->CurrentFrameTimestamp();
+  if (current_time.ToInternalValue() != 0) {
+    return current_time.InSecondsF();
   } else if (audio_renderer_.get()) {
     return audio_renderer_->GetCurrentRenderTime().InSecondsF();
   }
   return 0.0;
 }
 
 WebMediaPlayer::NetworkState WebMediaPlayerMS::networkState() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(1) << "WebMediaPlayerMS::networkState, state:" << network_state_;
   return network_state_;
@@ skipping 20 matching lines @@
   return true;
 }
 
 void WebMediaPlayerMS::paint(blink::WebCanvas* canvas,
                              const blink::WebRect& rect,
                              unsigned char alpha,
                              SkXfermode::Mode mode) {
   DVLOG(3) << "WebMediaPlayerMS::paint";
   DCHECK(thread_checker_.CalledOnValidThread());
 
+  auto frame = compositor_->GetCurrentFrame();

[DaleCurtis, 2015/08/17 21:31:29]

Inconistent use of auto. Use scoped_refptr<VideoFrame> here since you do so
elsehwere.

[qiangchen, 2015/08/18 01:21:49]

Done.

+
   media::Context3D context_3d;
-  if (current_frame_.get() && current_frame_->HasTextures()) {
+  if (frame.get() && frame->HasTextures()) {
     cc::ContextProvider* provider =
         RenderThreadImpl::current()->SharedMainThreadContextProvider().get();
     // GPU Process crashed.
     if (!provider)
       return;
     context_3d = media::Context3D(provider->ContextGL(), provider->GrContext());
     DCHECK(context_3d.gl);
   }
   gfx::RectF dest_rect(rect.x, rect.y, rect.width, rect.height);
-  video_renderer_.Paint(current_frame_, canvas, dest_rect, alpha, mode,
+  compositor_->VideoRenderer()->Paint(frame, canvas, dest_rect, alpha, mode,
                         media::VIDEO_ROTATION_0, context_3d);
-
-  {
-    base::AutoLock auto_lock(current_frame_lock_);
-    if (current_frame_.get())
-      current_frame_used_ = true;
-  }
 }
 
 bool WebMediaPlayerMS::hasSingleSecurityOrigin() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return true;
 }
 
 bool WebMediaPlayerMS::didPassCORSAccessCheck() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return true;
 }
 
 double WebMediaPlayerMS::mediaTimeForTimeValue(double timeValue) const {
   return media::ConvertSecondsToTimestamp(timeValue).InSecondsF();
 }
 
 unsigned WebMediaPlayerMS::decodedFrameCount() const {
   DCHECK(thread_checker_.CalledOnValidThread());
-  DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count_;
-  return total_frame_count_;
+  unsigned total_frame_count = compositor_->TotalFrameCount();
+  DVLOG(1) << "WebMediaPlayerMS::decodedFrameCount, " << total_frame_count;
+  return total_frame_count;
 }
 
 unsigned WebMediaPlayerMS::droppedFrameCount() const {
   DCHECK(thread_checker_.CalledOnValidThread());
-  DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count_;
-  return dropped_frame_count_;
+  unsigned dropped_frame_count = compositor_->DroppedFrameCount();
+  DVLOG(1) << "WebMediaPlayerMS::droppedFrameCount, " << dropped_frame_count;
+  return dropped_frame_count;
 }
 
 unsigned WebMediaPlayerMS::audioDecodedByteCount() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   NOTIMPLEMENTED();
   return 0;
 }
 
 unsigned WebMediaPlayerMS::videoDecodedByteCount() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   NOTIMPLEMENTED();
   return 0;
 }
 
 bool WebMediaPlayerMS::copyVideoTextureToPlatformTexture(
     blink::WebGraphicsContext3D* web_graphics_context,
     unsigned int texture,
     unsigned int internal_format,
     unsigned int type,
     bool premultiply_alpha,
     bool flip_y) {
   TRACE_EVENT0("media", "WebMediaPlayerMS:copyVideoTextureToPlatformTexture");
   DCHECK(thread_checker_.CalledOnValidThread());
 
-  scoped_refptr<media::VideoFrame> video_frame;
-  {
-    base::AutoLock auto_lock(current_frame_lock_);
-    video_frame = current_frame_;
-  }
+  scoped_refptr<media::VideoFrame> video_frame = compositor_->CurrentFrame();
 
   if (!video_frame.get() || video_frame->HasTextures() ||
       media::VideoFrame::NumPlanes(video_frame->format()) != 1) {
     return false;
   }
 
   // TODO(dshwang): need more elegant way to convert WebGraphicsContext3D to
   // GLES2Interface.
   gpu::gles2::GLES2Interface* gl =
       static_cast<gpu_blink::WebGraphicsContext3DImpl*>(web_graphics_context)
           ->GetGLInterface();
   media::SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
       gl, video_frame.get(), texture, internal_format, type, premultiply_alpha,
       flip_y);
   return true;
 }
 
-void WebMediaPlayerMS::SetVideoFrameProviderClient(
-    cc::VideoFrameProvider::Client* client) {
-  // This is called from both the main renderer thread and the compositor
-  // thread (when the main thread is blocked).
-  if (video_frame_provider_client_)
-    video_frame_provider_client_->StopUsingProvider();
-  video_frame_provider_client_ = client;
-}
-
-bool WebMediaPlayerMS::UpdateCurrentFrame(base::TimeTicks deadline_min,
-                                          base::TimeTicks deadline_max) {
-  // TODO(dalecurtis): This should make use of the deadline interval to ensure
-  // the painted frame is correct for the given interval.
-  NOTREACHED();
-  return false;
-}
-
-bool WebMediaPlayerMS::HasCurrentFrame() {
-  base::AutoLock auto_lock(current_frame_lock_);
-  return current_frame_;
-}
-
-scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
-  DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
-  base::AutoLock auto_lock(current_frame_lock_);
-  if (!current_frame_.get())
-    return NULL;
-  current_frame_used_ = true;
-  return current_frame_;
-}
-
-void WebMediaPlayerMS::PutCurrentFrame() {
-  DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
-}
-
 void WebMediaPlayerMS::OnFrameAvailable(
     const scoped_refptr<media::VideoFrame>& frame) {
   DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
   DCHECK(thread_checker_.CalledOnValidThread());
-  ++total_frame_count_;
+
+  base::TimeTicks render_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &render_time)) {
+    render_time = base::TimeTicks();
+  }
+  TRACE_EVENT1("webrtc", "WebMediaPlayerMS::OnFrameAvailable",
+               "Ideal Render Instant", render_time.ToInternalValue());
+
   if (!received_first_frame_) {
     received_first_frame_ = true;
-    {
-      base::AutoLock auto_lock(current_frame_lock_);
-      DCHECK(!current_frame_used_);
-      current_frame_ = frame;
-    }
     SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
     SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
-    GetClient()->sizeChanged();
 
     if (video_frame_provider_.get()) {
       video_weblayer_.reset(new cc_blink::WebLayerImpl(
-          cc::VideoLayer::Create(cc_blink::WebLayerImpl::LayerSettings(), this,
-                                 media::VIDEO_ROTATION_0)));
+          cc::VideoLayer::Create(cc_blink::WebLayerImpl::LayerSettings(),
+                                 compositor_.get(), media::VIDEO_ROTATION_0)));
       video_weblayer_->setOpaque(true);
       GetClient()->setWebLayer(video_weblayer_.get());
     }
   }
 
-  // Do not update |current_frame_| when paused.
-  if (paused_)
-    return;
+  bool size_changed = !compositor_->HasCurrentFrame() ||

[DaleCurtis, 2015/08/17 21:31:29]

No need for check since CurrentFrameSize returns gfx::Size() ?

[qiangchen, 2015/08/18 01:21:49]

Done.

+                      compositor_->CurrentFrameSize() != frame->natural_size();
 
-  bool size_changed = !current_frame_.get() ||
-                      current_frame_->natural_size() != frame->natural_size();
-
-  {
-    base::AutoLock auto_lock(current_frame_lock_);
-    if (!current_frame_used_ && current_frame_.get())
-      ++dropped_frame_count_;
-    current_frame_ = frame;
-    current_time_ = frame->timestamp();
-    current_frame_used_ = false;
-  }
+  compositor_->EnqueueFrame(frame);
 
   if (size_changed)
     GetClient()->sizeChanged();
-
-  GetClient()->repaint();
 }
 
 void WebMediaPlayerMS::RepaintInternal() {
   DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
   DCHECK(thread_checker_.CalledOnValidThread());
   GetClient()->repaint();
 }
 
 void WebMediaPlayerMS::OnSourceError() {
   DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
@@ skipping 15 matching lines @@
   // Always notify to ensure client has the latest value.
   GetClient()->readyStateChanged();
 }
 
 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(client_);
   return client_;
 }
 
+WebMediaPlayerMS::Compositor::Compositor(
+    scoped_refptr<base::SingleThreadTaskRunner> const& compositor_thread)
+    : compositor_thread_(compositor_thread),
+      video_frame_provider_client_(NULL),
+      current_frame_used_(false),
+      total_frame_count_(0),
+      dropped_frame_count_(0),
+      paused_(false) {}
+
+WebMediaPlayerMS::Compositor::~Compositor() {
+  DCHECK(compositor_thread_->BelongsToCurrentThread());
+  if (video_frame_provider_client_)
+    video_frame_provider_client_->StopUsingProvider();
+}
+
+void WebMediaPlayerMS::Compositor::EnqueueFrame(
+    scoped_refptr<media::VideoFrame> const& frame) {
+  base::AutoLock auto_lock(current_frame_lock_);
+  ++total_frame_count_;
+  if (!current_frame_used_)
+    ++dropped_frame_count_;
+
+  // When paused, do not update current_frame;
+  // TODO(qiangchen): After applying VRA, this check should move to
+  // UpdateCurrentFrame. During pause, we have to let frames go through VRA and
+  // then be thrown away, otherwise VRA will think there is a very long frame.
+  if (paused_)
+    return;
+
+  if (base::TimeTicks::Now() > last_deadline_max_) {
+    // TODO(qiangchen): This shows vsyncs stops rendering frames. A probable
+    // cause is that the tab is not in the front. But we still have to let
+    // old frames go. Call VRA::RemoveExpiredFrames.
+  }
+
+  // TODO(qiangchen): Instead of using one variable to hold one frame, use
+  // VideoRendererAlgorithm.
+  current_frame_ = frame;
+  current_frame_used_ = false;
+  current_time_ = frame->timestamp();
+}
+
+bool WebMediaPlayerMS::Compositor::UpdateCurrentFrame(
+    base::TimeTicks deadline_min,
+    base::TimeTicks deadline_max) {
+  DCHECK(compositor_thread_->BelongsToCurrentThread());
+  base::AutoLock auto_lock(current_frame_lock_);
+  TRACE_EVENT_BEGIN2("webrtc", "WebMediaPlayerMS::UpdateCurrentFrame",
+                     "Actual Render Begin", deadline_min.ToInternalValue(),
+                     "Actual Render End", deadline_max.ToInternalValue());
+  last_deadline_max_ = deadline_max;
+
+  // TODO(dalecurtis): This should make use of the deadline interval to ensure
+  // the painted frame is correct for the given interval.
+
+  base::TimeTicks render_time;
+  if (!current_frame_->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &render_time)) {
+    render_time = base::TimeTicks();
+  }
+  TRACE_EVENT_END1("webrtc", "WebMediaPlayerMS::UpdateCurrentFrame",
+                   "Ideal Render Instant", render_time.ToInternalValue());
+  return !current_frame_used_;
+}
+
+bool WebMediaPlayerMS::Compositor::HasCurrentFrame() {
+  base::AutoLock auto_lock(current_frame_lock_);
+  return current_frame_;

[DaleCurtis, 2015/08/17 21:31:29]

Might need !! to avoid compilation errors on windows, I forget.

[qiangchen, 2015/08/18 01:21:49]

Done.

+}
+
+scoped_refptr<media::VideoFrame>
+WebMediaPlayerMS::Compositor::GetCurrentFrame() {
+  DVLOG(3) << "WebMediaPlayerMS::Compositor::GetCurrentFrame";
+  base::AutoLock auto_lock(current_frame_lock_);
+  if (!current_frame_.get())

[DaleCurtis, 2015/08/17 21:31:29]

Conditional necessary? If current frame is null, does it matter if we set
current_frame_used_ to true? You might need a !current_frame and
!current_frame_used above.

[qiangchen, 2015/08/18 01:21:49]

N/A

+    return NULL;
+  current_frame_used_ = true;
+  return current_frame_;
+}
+
+void WebMediaPlayerMS::Compositor::PutCurrentFrame() {
+  DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
+}
+
+void WebMediaPlayerMS::Compositor::SetVideoFrameProviderClient(
+    cc::VideoFrameProvider::Client* client) {
+  DCHECK(compositor_thread_->BelongsToCurrentThread());
+  if (video_frame_provider_client_)
+    video_frame_provider_client_->StopUsingProvider();
+
+  video_frame_provider_client_ = client;
+  if (video_frame_provider_client_)
+    video_frame_provider_client_->StartRendering();
+}
+
+void WebMediaPlayerMS::Compositor::StartRendering() {
+  DCHECK(compositor_thread_->BelongsToCurrentThread());
+  paused_ = false;
+  if (video_frame_provider_client_)
+    video_frame_provider_client_->StartRendering();
+}
+
+void WebMediaPlayerMS::Compositor::StopRendering() {
+  DCHECK(compositor_thread_->BelongsToCurrentThread());
+  paused_ = true;
+  if (video_frame_provider_client_)
+    video_frame_provider_client_->StopRendering();
+
+  base::AutoLock auto_lock(current_frame_lock_);
+  if (!current_frame_.get())
+    return;
+
+  // Copy the frame so that rendering can show the last received frame.
+  // The original frame must not be referenced when the player is paused since
+  // there might be a finite number of available buffers. E.g, video that
+  // originates from a video camera.
+  scoped_refptr<media::VideoFrame> new_frame =
+      CopyFrameToYV12(current_frame_, &video_renderer_);
+
+  current_frame_ = new_frame;
+}
+
+gfx::Size WebMediaPlayerMS::Compositor::CurrentFrameSize() {
+  base::AutoLock auto_lock(current_frame_lock_);
+  if (current_frame_.get())

[DaleCurtis, 2015/08/17 21:31:29]

Ternary would be cleaner.

[qiangchen, 2015/08/18 01:21:49]

Done.

+    return current_frame_->natural_size();
+  return gfx::Size();
+}
+
+base::TimeDelta WebMediaPlayerMS::Compositor::CurrentFrameTimestamp() {
+  return current_time_;
+}
+
+unsigned WebMediaPlayerMS::Compositor::TotalFrameCount() {
+  return total_frame_count_;
+}
+
+unsigned WebMediaPlayerMS::Compositor::DroppedFrameCount() {
+  return dropped_frame_count_;
+}
+
+scoped_refptr<media::VideoFrame> WebMediaPlayerMS::Compositor::CurrentFrame() {
+  base::AutoLock auto_lock(current_frame_lock_);
+  if (!current_frame_.get())

[DaleCurtis, 2015/08/17 21:31:29]

Should just delete this method after switching to counting dropped frames as
I've mentioned in my previous comment.

[qiangchen, 2015/08/18 01:21:49]

Done.

+    return NULL;
+  return current_frame_;
+}
+
+media::SkCanvasVideoRenderer* WebMediaPlayerMS::Compositor::VideoRenderer() {
+  return &video_renderer_;
+}
 }  // namespace content