Make rendering of MediaStreams reflect changes to its set of tracks.

content/renderer/media/webmediaplayer_ms.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webmediaplayer_ms.h"
 
 #include <stddef.h>
 #include <limits>
 #include <string>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "build/build_config.h"
 #include "cc/blink/web_layer_impl.h"
 #include "cc/layers/video_frame_provider_client_impl.h"
 #include "cc/layers/video_layer.h"
 #include "content/child/child_process.h"
 #include "content/public/renderer/media_stream_audio_renderer.h"
 #include "content/public/renderer/media_stream_renderer_factory.h"
 #include "content/public/renderer/media_stream_video_renderer.h"
+#include "content/renderer/media/media_stream_audio_track.h"
+#include "content/renderer/media/media_stream_video_track.h"
 #include "content/renderer/media/web_media_element_source_utils.h"
 #include "content/renderer/media/webmediaplayer_ms_compositor.h"
 #include "content/renderer/render_frame_impl.h"
 #include "content/renderer/render_thread_impl.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/media_content_type.h"
 #include "media/base/media_log.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_rotation.h"
 #include "media/base/video_types.h"
 #include "media/blink/webmediaplayer_util.h"
 #include "services/ui/public/cpp/gpu/context_provider_command_buffer.h"
 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
 #include "third_party/WebKit/public/platform/WebMediaPlayerSource.h"
 #include "third_party/WebKit/public/platform/WebRect.h"
 #include "third_party/WebKit/public/platform/WebSize.h"
 #include "third_party/WebKit/public/web/WebLocalFrame.h"
 
+namespace {
+enum class RendererReloadAction {
+  KEEP_RENDERER,
+  REMOVE_RENDERER,
+  NEW_RENDERER
+};
+}  // namespace
+
 namespace content {
 
 // FrameDeliverer is responsible for delivering frames received on
 // the IO thread by calling of EnqueueFrame() method of |compositor_|.
 //
 // It is created on the main thread, but methods should be called and class
 // should be destructed on the IO thread.
 class WebMediaPlayerMS::FrameDeliverer {
  public:
   FrameDeliverer(const base::WeakPtr<WebMediaPlayerMS>& player,
@@ skipping 132 matching lines @@
   delegate_id_ = delegate_->AddObserver(this);
 
   media_log_->AddEvent(
       media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED));
 }
 
 WebMediaPlayerMS::~WebMediaPlayerMS() {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
 
+  if (!web_stream_.IsNull()) {
+    MediaStream* native_stream = MediaStream::GetMediaStream(web_stream_);
+    if (native_stream)
+      native_stream->RemoveObserver(this);
+  }
+
   // Destruct compositor resources in the proper order.
   get_client()->SetWebLayer(nullptr);
   if (video_weblayer_)
     static_cast<cc::VideoLayer*>(video_weblayer_->layer())->StopUsingProvider();
 
   if (frame_deliverer_)
     io_task_runner_->DeleteSoon(FROM_HERE, frame_deliverer_.release());
 
   if (compositor_)
     compositor_->StopUsingProvider();
@@ skipping 13 matching lines @@
 
 void WebMediaPlayerMS::Load(LoadType load_type,
                             const blink::WebMediaPlayerSource& source,
                             CORSMode /*cors_mode*/) {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
 
   // TODO(acolwell): Change this to DCHECK_EQ(load_type, LoadTypeMediaStream)
   // once Blink-side changes land.
   DCHECK_NE(load_type, kLoadTypeMediaSource);
-  blink::WebMediaStream web_stream =
-      GetWebMediaStreamFromWebMediaPlayerSource(source);
+  web_stream_ = GetWebMediaStreamFromWebMediaPlayerSource(source);

[DaleCurtis, 2017/07/31 16:22:40]

How can this be null now? I don't see a change for that.

[Guido Urdaneta, 2017/07/31 16:30:23]

Null check in line 235 from the original file suggests that it can be null.

[DaleCurtis, 2017/07/31 16:31:19]

Ah missed that, thanks!

+  if (!web_stream_.IsNull()) {
+    MediaStream* native_stream = MediaStream::GetMediaStream(web_stream_);
+    if (native_stream)
+      native_stream->AddObserver(this);
+  }
 
   compositor_ = new WebMediaPlayerMSCompositor(
-      compositor_task_runner_, io_task_runner_, web_stream, AsWeakPtr());
+      compositor_task_runner_, io_task_runner_, web_stream_, AsWeakPtr());
 
   SetNetworkState(WebMediaPlayer::kNetworkStateLoading);
   SetReadyState(WebMediaPlayer::kReadyStateHaveNothing);
   std::string stream_id =
-      web_stream.IsNull() ? std::string() : web_stream.Id().Utf8();
+      web_stream_.IsNull() ? std::string() : web_stream_.Id().Utf8();
   media_log_->AddEvent(media_log_->CreateLoadEvent(stream_id));
 
   frame_deliverer_.reset(new WebMediaPlayerMS::FrameDeliverer(
       AsWeakPtr(),
       base::Bind(&WebMediaPlayerMSCompositor::EnqueueFrame, compositor_)));
   video_frame_provider_ = renderer_factory_->GetVideoRenderer(
-      web_stream, media::BindToCurrentLoop(base::Bind(
-                      &WebMediaPlayerMS::OnSourceError, AsWeakPtr())),
+      web_stream_,
+      media::BindToCurrentLoop(
+          base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr())),
       frame_deliverer_->GetRepaintCallback(), io_task_runner_,
       media_task_runner_, worker_task_runner_, gpu_factories_);
 
   RenderFrame* const frame = RenderFrame::FromWebFrame(frame_);
 
   if (frame) {
     // Report UMA and RAPPOR metrics.
     GURL url = source.IsURL() ? GURL(source.GetAsURL()) : GURL();
     media::ReportMetrics(load_type, url, frame_->GetSecurityOrigin(),
                          media_log_.get());
 
     audio_renderer_ = renderer_factory_->GetAudioRenderer(
-        web_stream, frame->GetRoutingID(), initial_audio_output_device_id_,
+        web_stream_, frame->GetRoutingID(), initial_audio_output_device_id_,
         initial_security_origin_);
   }
 
   if (!video_frame_provider_ && !audio_renderer_) {
     SetNetworkState(WebMediaPlayer::kNetworkStateNetworkError);
     return;
   }
 
   if (audio_renderer_) {
     audio_renderer_->SetVolume(volume_);
     audio_renderer_->Start();
+
+    // Store the ID of audio track being played in |current_video_track_id_|
+
+    blink::WebVector<blink::WebMediaStreamTrack> audio_tracks;
+    if (!web_stream_.IsNull()) {
+      web_stream_.AudioTracks(audio_tracks);
+      DCHECK_GT(audio_tracks.size(), 0U);
+      current_audio_track_id_ = audio_tracks[0].Id();
+    }
   }
-  if (video_frame_provider_)
+  if (video_frame_provider_) {
     video_frame_provider_->Start();
+
+    // Store the ID of video track being played in |current_video_track_id_|
+    if (!web_stream_.IsNull()) {
+      blink::WebVector<blink::WebMediaStreamTrack> video_tracks;
+      web_stream_.VideoTracks(video_tracks);
+      DCHECK_GT(video_tracks.size(), 0U);
+      current_video_track_id_ = video_tracks[0].Id();
+    }
+  }
   if (audio_renderer_ && !video_frame_provider_) {
     // This is audio-only mode.
     SetReadyState(WebMediaPlayer::kReadyStateHaveMetadata);
     SetReadyState(WebMediaPlayer::kReadyStateHaveEnoughData);
   }
 }
 
+void WebMediaPlayerMS::TrackAdded(const blink::WebMediaStreamTrack& track) {
+  Reload();
+}
+
+void WebMediaPlayerMS::TrackRemoved(const blink::WebMediaStreamTrack& track) {
+  Reload();
+}
+
+void WebMediaPlayerMS::Reload() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (web_stream_.IsNull())
+    return;
+
+  ReloadVideo();
+  ReloadAudio();
+}
+
+void WebMediaPlayerMS::ReloadVideo() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!web_stream_.IsNull());
+  blink::WebVector<blink::WebMediaStreamTrack> video_tracks;
+  // VideoTracks() is a getter.
+  web_stream_.VideoTracks(video_tracks);
+
+  RendererReloadAction renderer_action = RendererReloadAction::KEEP_RENDERER;
+  if (video_tracks.IsEmpty()) {
+    if (video_frame_provider_)
+      renderer_action = RendererReloadAction::REMOVE_RENDERER;
+    current_video_track_id_ = blink::WebString();
+  } else if (video_tracks[0].Id() != current_video_track_id_) {
+    renderer_action = RendererReloadAction::NEW_RENDERER;
+    current_video_track_id_ = video_tracks[0].Id();
+  }
+
+  switch (renderer_action) {
+    case RendererReloadAction::NEW_RENDERER:
+      if (video_frame_provider_)
+        video_frame_provider_->Stop();
+
+      video_frame_provider_ = renderer_factory_->GetVideoRenderer(
+          web_stream_,
+          media::BindToCurrentLoop(
+              base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr())),
+          frame_deliverer_->GetRepaintCallback(), io_task_runner_,
+          media_task_runner_, worker_task_runner_, gpu_factories_);
+      DCHECK(video_frame_provider_);
+      video_frame_provider_->Start();
+      break;
+
+    case RendererReloadAction::REMOVE_RENDERER:
+      video_frame_provider_->Stop();
+      video_frame_provider_ = nullptr;
+      break;
+
+    default:
+      return;
+  }
+
+  DCHECK_NE(renderer_action, RendererReloadAction::KEEP_RENDERER);
+  if (!paused_)
+    delegate_->DidPlayerSizeChange(delegate_id_, NaturalSize());
+}
+
+void WebMediaPlayerMS::ReloadAudio() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!web_stream_.IsNull());
+  RenderFrame* const frame = RenderFrame::FromWebFrame(frame_);
+  if (!frame)
+    return;
+
+  blink::WebVector<blink::WebMediaStreamTrack> audio_tracks;
+  // AudioTracks() is a getter.
+  web_stream_.AudioTracks(audio_tracks);
+
+  RendererReloadAction renderer_action = RendererReloadAction::KEEP_RENDERER;
+  if (audio_tracks.IsEmpty()) {
+    if (audio_renderer_)
+      renderer_action = RendererReloadAction::REMOVE_RENDERER;
+    current_audio_track_id_ = blink::WebString();
+  } else if (audio_tracks[0].Id() != current_video_track_id_) {
+    renderer_action = RendererReloadAction::NEW_RENDERER;
+    current_audio_track_id_ = audio_tracks[0].Id();
+  }
+
+  switch (renderer_action) {
+    case RendererReloadAction::NEW_RENDERER:
+      if (audio_renderer_)
+        audio_renderer_->Stop();
+
+      audio_renderer_ = renderer_factory_->GetAudioRenderer(
+          web_stream_, frame->GetRoutingID(), initial_audio_output_device_id_,
+          initial_security_origin_);
+      audio_renderer_->SetVolume(volume_);
+      audio_renderer_->Start();
+      audio_renderer_->Play();
+      break;
+
+    case RendererReloadAction::REMOVE_RENDERER:
+      audio_renderer_->Stop();
+      audio_renderer_ = nullptr;
+      break;
+
+    default:
+      break;
+  }
+}
+
 void WebMediaPlayerMS::Play() {
   DVLOG(1) << __func__;
   DCHECK(thread_checker_.CalledOnValidThread());
 
   media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY));
   if (!paused_)
     return;
 
   if (video_frame_provider_)
     video_frame_provider_->Resume();
@@ skipping 86 matching lines @@
   return (video_frame_provider_.get() != nullptr);
 }
 
 bool WebMediaPlayerMS::HasAudio() const {
   DCHECK(thread_checker_.CalledOnValidThread());
   return (audio_renderer_.get() != nullptr);
 }
 
 blink::WebSize WebMediaPlayerMS::NaturalSize() const {
   DCHECK(thread_checker_.CalledOnValidThread());
+  if (!video_frame_provider_)
+    return blink::WebSize();
+
   if (video_rotation_ == media::VIDEO_ROTATION_90 ||
       video_rotation_ == media::VideoRotation::VIDEO_ROTATION_270) {
     const gfx::Size& current_size = compositor_->GetCurrentSize();
     return blink::WebSize(current_size.height(), current_size.width());
   }
   return blink::WebSize(compositor_->GetCurrentSize());
 }
 
 bool WebMediaPlayerMS::Paused() const {
   DCHECK(thread_checker_.CalledOnValidThread());
@@ skipping 327 matching lines @@
   video_renderer_.ResetCache();
 }
 
 void WebMediaPlayerMS::TriggerResize() {
   get_client()->SizeChanged();
 
   delegate_->DidPlayerSizeChange(delegate_id_, NaturalSize());
 }
 
 }  // namespace content