Preliminary change for new rtc rendering algorithm

content/renderer/media/webmediaplayer_ms.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webmediaplayer_ms.h"
 
 #include <limits>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "cc/blink/context_provider_web_context.h"
 #include "cc/blink/web_layer_impl.h"
+#include "cc/layers/video_frame_provider_client_impl.h"
 #include "cc/layers/video_layer.h"
 #include "content/public/renderer/media_stream_audio_renderer.h"
 #include "content/public/renderer/media_stream_renderer_factory.h"
 #include "content/public/renderer/render_view.h"
 #include "content/public/renderer/video_frame_provider.h"
 #include "content/renderer/render_frame_impl.h"
 #include "content/renderer/render_thread_impl.h"
 #include "gpu/blink/webgraphicscontext3d_impl.h"
 #include "media/base/media_log.h"
 #include "media/base/video_frame.h"
@@ skipping 61 matching lines @@
   return new_frame;
 }
 
 }  // anonymous namespace
 
 WebMediaPlayerMS::WebMediaPlayerMS(
     blink::WebFrame* frame,
     blink::WebMediaPlayerClient* client,
     base::WeakPtr<media::WebMediaPlayerDelegate> delegate,
     media::MediaLog* media_log,
-    scoped_ptr<MediaStreamRendererFactory> factory)
+    scoped_ptr<MediaStreamRendererFactory> factory,
+    scoped_refptr<base::SingleThreadTaskRunner> compositor_thread)
     : frame_(frame),
       network_state_(WebMediaPlayer::NetworkStateEmpty),
       ready_state_(WebMediaPlayer::ReadyStateHaveNothing),
       buffered_(static_cast<size_t>(0)),
       volume_(1.0f),
       client_(client),
       delegate_(delegate),
       paused_(true),
       current_frame_used_(false),
       video_frame_provider_client_(NULL),
       received_first_frame_(false),
       total_frame_count_(0),
       dropped_frame_count_(0),
       media_log_(media_log),
-      renderer_factory_(factory.Pass()) {
+      renderer_factory_(factory.Pass()),
+      compositor_thread_(compositor_thread),
+      wait_event_(false,false) {
   DVLOG(1) << "WebMediaPlayerMS::ctor";
   media_log_->AddEvent(
       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
 }
 
 WebMediaPlayerMS::~WebMediaPlayerMS() {
   DVLOG(1) << "WebMediaPlayerMS::dtor";
   DCHECK(thread_checker_.CalledOnValidThread());
 
-  SetVideoFrameProviderClient(NULL);
+  // Wait till stop procedure finishes to avoid the case that after
+  // WebMediaPlayerMS is destroyed, VideoFrameProvider is still trying to access
+  // it.
+
+  if (video_frame_provider_client_){
+    compositor_thread_->PostTask(
+            FROM_HERE,
+            base::Bind(
+                &WebMediaPlayerMS::StopVideoFrameProviderAndWakeUpHelper,
+                video_frame_provider_client_,
+                &wait_event_));
+    wait_event_.Wait();

[DaleCurtis, 2015/08/13 18:06:53]

Hmm, this is unfortunate. Typically you'd use CompositorTaskRunner::DeleteSoon()
to avoid this wait -- which is what WMPI does with the VideoFrameCompositor
object.

Seems like it might be better for you to do something similar here, have an
internal class which manages running on the compositor thread and uses a lock to
clear the state on the render thread. You can then pass that object to the
compositor thread for deletion.

I.e., just move all your helpers into an inner class and add a lock.

[qiangchen, 2015/08/14 16:52:50]

Done.

+  }
+
   GetClient()->setWebLayer(NULL);
 
   if (video_frame_provider_.get())
     video_frame_provider_->Stop();
 
   if (audio_renderer_.get())
     audio_renderer_->Stop();
 
   media_log_->AddEvent(
       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_DESTROYED));
@@ skipping 48 matching lines @@
 }
 
 void WebMediaPlayerMS::play() {
   DVLOG(1) << "WebMediaPlayerMS::play";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (paused_) {
     if (video_frame_provider_.get())
       video_frame_provider_->Play();
 
+    if (video_frame_provider_client_){
+      compositor_thread_->PostTask(
+          FROM_HERE,
+          base::Bind(&WebMediaPlayerMS::StartRenderingHelper,
+                     video_frame_provider_client_));
+    }
+
     if (audio_renderer_.get())
       audio_renderer_->Play();
 
     if (delegate_.get())
       delegate_->DidPlay(this);
   }
 
   paused_ = false;
 
   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
 }
 
 void WebMediaPlayerMS::pause() {
   DVLOG(1) << "WebMediaPlayerMS::pause";
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (video_frame_provider_.get())
     video_frame_provider_->Pause();
 
+  if (video_frame_provider_client_){
+    compositor_thread_->PostTask(
+        FROM_HERE,
+        base::Bind(&WebMediaPlayerMS::StopRenderingHelper,
+                   video_frame_provider_client_));
+  }
+
   if (!paused_) {
     if (audio_renderer_.get())
       audio_renderer_->Pause();
 
     if (delegate_.get())
       delegate_->DidPause(this);
   }
 
   paused_ = true;
 
@@ skipping 223 matching lines @@
       gl, video_frame.get(), texture, internal_format, type, premultiply_alpha,
       flip_y);
   return true;
 }
 
 void WebMediaPlayerMS::SetVideoFrameProviderClient(
     cc::VideoFrameProvider::Client* client) {
   // This is called from both the main renderer thread and the compositor
   // thread (when the main thread is blocked).
   if (video_frame_provider_client_)
-    video_frame_provider_client_->StopUsingProvider();
+    compositor_thread_->PostTask(
+        FROM_HERE,
+        base::Bind(
+            &WebMediaPlayerMS::StopVideoFrameProviderHelper,
+            video_frame_provider_client_));
+
   video_frame_provider_client_ = client;
+  if (client && !paused_)
+    compositor_thread_->PostTask(
+        FROM_HERE,
+        base::Bind(&WebMediaPlayerMS::StartRenderingHelper,
+                   video_frame_provider_client_));
+}
+
+void WebMediaPlayerMS::StopVideoFrameProviderAndWakeUpHelper(
+    cc::VideoFrameProvider::Client* client,
+    base::WaitableEvent* event) {
+  client->StopUsingProvider();
+  event->Signal();
+}
+
+void WebMediaPlayerMS::StopVideoFrameProviderHelper(
+    cc::VideoFrameProvider::Client* client) {
+  client->StopUsingProvider();
+}
+
+void WebMediaPlayerMS::StartRenderingHelper(
+    cc::VideoFrameProvider::Client* client) {
+  client->StartRendering();
+}
+
+void WebMediaPlayerMS::StopRenderingHelper(
+    cc::VideoFrameProvider::Client* client) {
+  client->StopRendering();
 }
 
 bool WebMediaPlayerMS::UpdateCurrentFrame(base::TimeTicks deadline_min,
                                           base::TimeTicks deadline_max) {
+  TRACE_EVENT_BEGIN2("webrtc", "WebMediaPlayerMS::UpdateCurrentFrame",
+                     "Actual Render Begin", deadline_min.ToInternalValue(),
+                     "Actual Render End", deadline_max.ToInternalValue());
+  last_deadline_max_ = deadline_max;
+
   // TODO(dalecurtis): This should make use of the deadline interval to ensure
   // the painted frame is correct for the given interval.
-  NOTREACHED();
-  return false;
+
+  base::TimeTicks render_time;
+  if (!current_frame_->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &render_time)) {
+    render_time = base::TimeTicks();
+  }
+  TRACE_EVENT_END1("webrtc", "WebMediaPlayerMS::UpdateCurrentFrame",
+                   "Ideal Render Instant", render_time.ToInternalValue());
+  return !current_frame_used_;
 }
 
 bool WebMediaPlayerMS::HasCurrentFrame() {
   base::AutoLock auto_lock(current_frame_lock_);
   return current_frame_;
 }
 
 scoped_refptr<media::VideoFrame> WebMediaPlayerMS::GetCurrentFrame() {
   DVLOG(3) << "WebMediaPlayerMS::GetCurrentFrame";
   base::AutoLock auto_lock(current_frame_lock_);
   if (!current_frame_.get())
     return NULL;
   current_frame_used_ = true;
   return current_frame_;
 }
 
 void WebMediaPlayerMS::PutCurrentFrame() {
   DVLOG(3) << "WebMediaPlayerMS::PutCurrentFrame";
 }
 
 void WebMediaPlayerMS::OnFrameAvailable(
     const scoped_refptr<media::VideoFrame>& frame) {
   DVLOG(3) << "WebMediaPlayerMS::OnFrameAvailable";
   DCHECK(thread_checker_.CalledOnValidThread());
+
+  base::TimeTicks render_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &render_time)) {
+    render_time = base::TimeTicks();
+  }
+  TRACE_EVENT1("webrtc", "WebMediaPlayerMS::OnFrameAvailable",
+               "Ideal Render Instant", render_time.ToInternalValue());
+
   ++total_frame_count_;
   if (!received_first_frame_) {
     received_first_frame_ = true;
     {
       base::AutoLock auto_lock(current_frame_lock_);
       DCHECK(!current_frame_used_);
       current_frame_ = frame;
     }
     SetReadyState(WebMediaPlayer::ReadyStateHaveMetadata);
     SetReadyState(WebMediaPlayer::ReadyStateHaveEnoughData);
     GetClient()->sizeChanged();
 
     if (video_frame_provider_.get()) {
       video_weblayer_.reset(new cc_blink::WebLayerImpl(
           cc::VideoLayer::Create(cc_blink::WebLayerImpl::LayerSettings(), this,
                                  media::VIDEO_ROTATION_0)));
       video_weblayer_->setOpaque(true);
       GetClient()->setWebLayer(video_weblayer_.get());
     }
   }
 
   // Do not update |current_frame_| when paused.
   if (paused_)
     return;
 
+  if (base::TimeTicks::Now() > last_deadline_max_){
+    // TODO(qiangchen): Theoretically Now is between
+    // [last_deadline_min_ - vsync_duration, last_deadline_min]. If Now is later
+    // than last_deadline_max, it means UpdateCurrentFrame does not get called
+    // for 2 vsyncs. A probable cause is that the video tag is invisible, but we
+    // have to let old frames go (VRA::RemoveExpiredFrames), otherwise frames
+    // will pile up, and when we use up all frame buffers, decoder will be
+    // stuck.
+  }
+
   bool size_changed = !current_frame_.get() ||
                       current_frame_->natural_size() != frame->natural_size();
 
   {
     base::AutoLock auto_lock(current_frame_lock_);
     if (!current_frame_used_ && current_frame_.get())
       ++dropped_frame_count_;
     current_frame_ = frame;
     current_time_ = frame->timestamp();
     current_frame_used_ = false;
   }
 
   if (size_changed)
     GetClient()->sizeChanged();
-
-  GetClient()->repaint();
 }
 
 void WebMediaPlayerMS::RepaintInternal() {
   DVLOG(1) << "WebMediaPlayerMS::RepaintInternal";
   DCHECK(thread_checker_.CalledOnValidThread());
   GetClient()->repaint();
 }
 
 void WebMediaPlayerMS::OnSourceError() {
   DVLOG(1) << "WebMediaPlayerMS::OnSourceError";
@@ skipping 16 matching lines @@
   GetClient()->readyStateChanged();
 }
 
 blink::WebMediaPlayerClient* WebMediaPlayerMS::GetClient() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(client_);
   return client_;
 }
 
 }  // namespace content