OLD | NEW |
1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/renderer/media/webmediaplayer_ms.h" | 5 #include "content/renderer/media/webmediaplayer_ms.h" |
6 | 6 |
7 #include <stddef.h> | 7 #include <stddef.h> |
8 #include <limits> | 8 #include <limits> |
9 #include <string> | 9 #include <string> |
10 #include <utility> | 10 #include <utility> |
11 | 11 |
12 #include "base/bind.h" | 12 #include "base/bind.h" |
13 #include "base/callback.h" | 13 #include "base/callback.h" |
14 #include "base/threading/thread_task_runner_handle.h" | 14 #include "base/threading/thread_task_runner_handle.h" |
15 #include "build/build_config.h" | 15 #include "build/build_config.h" |
16 #include "cc/blink/web_layer_impl.h" | 16 #include "cc/blink/web_layer_impl.h" |
17 #include "cc/layers/video_frame_provider_client_impl.h" | 17 #include "cc/layers/video_frame_provider_client_impl.h" |
18 #include "cc/layers/video_layer.h" | 18 #include "cc/layers/video_layer.h" |
19 #include "content/child/child_process.h" | 19 #include "content/child/child_process.h" |
20 #include "content/public/renderer/media_stream_audio_renderer.h" | 20 #include "content/public/renderer/media_stream_audio_renderer.h" |
21 #include "content/public/renderer/media_stream_renderer_factory.h" | 21 #include "content/public/renderer/media_stream_renderer_factory.h" |
22 #include "content/public/renderer/media_stream_video_renderer.h" | 22 #include "content/public/renderer/media_stream_video_renderer.h" |
| 23 #include "content/renderer/media/media_stream_audio_track.h" |
| 24 #include "content/renderer/media/media_stream_video_track.h" |
23 #include "content/renderer/media/web_media_element_source_utils.h" | 25 #include "content/renderer/media/web_media_element_source_utils.h" |
24 #include "content/renderer/media/webmediaplayer_ms_compositor.h" | 26 #include "content/renderer/media/webmediaplayer_ms_compositor.h" |
25 #include "content/renderer/media/webrtc_logging.h" | 27 #include "content/renderer/media/webrtc_logging.h" |
26 #include "content/renderer/render_frame_impl.h" | 28 #include "content/renderer/render_frame_impl.h" |
27 #include "content/renderer/render_thread_impl.h" | 29 #include "content/renderer/render_thread_impl.h" |
28 #include "media/base/bind_to_current_loop.h" | 30 #include "media/base/bind_to_current_loop.h" |
29 #include "media/base/media_content_type.h" | 31 #include "media/base/media_content_type.h" |
30 #include "media/base/media_log.h" | 32 #include "media/base/media_log.h" |
31 #include "media/base/video_frame.h" | 33 #include "media/base/video_frame.h" |
32 #include "media/base/video_rotation.h" | 34 #include "media/base/video_rotation.h" |
33 #include "media/base/video_types.h" | 35 #include "media/base/video_types.h" |
34 #include "media/blink/webmediaplayer_util.h" | 36 #include "media/blink/webmediaplayer_util.h" |
35 #include "services/ui/public/cpp/gpu/context_provider_command_buffer.h" | 37 #include "services/ui/public/cpp/gpu/context_provider_command_buffer.h" |
36 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h" | 38 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h" |
37 #include "third_party/WebKit/public/platform/WebMediaPlayerSource.h" | 39 #include "third_party/WebKit/public/platform/WebMediaPlayerSource.h" |
38 #include "third_party/WebKit/public/platform/WebRect.h" | 40 #include "third_party/WebKit/public/platform/WebRect.h" |
39 #include "third_party/WebKit/public/platform/WebSize.h" | 41 #include "third_party/WebKit/public/platform/WebSize.h" |
40 #include "third_party/WebKit/public/web/WebLocalFrame.h" | 42 #include "third_party/WebKit/public/web/WebLocalFrame.h" |
41 | 43 |
| 44 namespace { |
| 45 enum class RendererReloadAction { |
| 46 KEEP_RENDERER, |
| 47 REMOVE_RENDERER, |
| 48 NEW_RENDERER |
| 49 }; |
| 50 } // namespace |
| 51 |
42 namespace content { | 52 namespace content { |
43 | 53 |
44 // FrameDeliverer is responsible for delivering frames received on | 54 // FrameDeliverer is responsible for delivering frames received on |
45 // the IO thread by calling of EnqueueFrame() method of |compositor_|. | 55 // the IO thread by calling of EnqueueFrame() method of |compositor_|. |
46 // | 56 // |
47 // It is created on the main thread, but methods should be called and class | 57 // It is created on the main thread, but methods should be called and class |
48 // should be destructed on the IO thread. | 58 // should be destructed on the IO thread. |
49 class WebMediaPlayerMS::FrameDeliverer { | 59 class WebMediaPlayerMS::FrameDeliverer { |
50 public: | 60 public: |
51 FrameDeliverer(const base::WeakPtr<WebMediaPlayerMS>& player, | 61 FrameDeliverer(const base::WeakPtr<WebMediaPlayerMS>& player, |
(...skipping 132 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
184 delegate_id_ = delegate_->AddObserver(this); | 194 delegate_id_ = delegate_->AddObserver(this); |
185 | 195 |
186 media_log_->AddEvent( | 196 media_log_->AddEvent( |
187 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED)); | 197 media_log_->CreateEvent(media::MediaLogEvent::WEBMEDIAPLAYER_CREATED)); |
188 } | 198 } |
189 | 199 |
190 WebMediaPlayerMS::~WebMediaPlayerMS() { | 200 WebMediaPlayerMS::~WebMediaPlayerMS() { |
191 DVLOG(1) << __func__; | 201 DVLOG(1) << __func__; |
192 DCHECK(thread_checker_.CalledOnValidThread()); | 202 DCHECK(thread_checker_.CalledOnValidThread()); |
193 | 203 |
| 204 if (!web_stream_.IsNull()) { |
| 205 MediaStream* native_stream = MediaStream::GetMediaStream(web_stream_); |
| 206 if (native_stream) |
| 207 native_stream->RemoveObserver(this); |
| 208 } |
| 209 |
194 // Destruct compositor resources in the proper order. | 210 // Destruct compositor resources in the proper order. |
195 get_client()->SetWebLayer(nullptr); | 211 get_client()->SetWebLayer(nullptr); |
196 if (video_weblayer_) | 212 if (video_weblayer_) |
197 static_cast<cc::VideoLayer*>(video_weblayer_->layer())->StopUsingProvider(); | 213 static_cast<cc::VideoLayer*>(video_weblayer_->layer())->StopUsingProvider(); |
198 | 214 |
199 if (frame_deliverer_) | 215 if (frame_deliverer_) |
200 io_task_runner_->DeleteSoon(FROM_HERE, frame_deliverer_.release()); | 216 io_task_runner_->DeleteSoon(FROM_HERE, frame_deliverer_.release()); |
201 | 217 |
202 if (compositor_) | 218 if (compositor_) |
203 compositor_->StopUsingProvider(); | 219 compositor_->StopUsingProvider(); |
(...skipping 13 matching lines...) Expand all Loading... |
217 | 233 |
218 void WebMediaPlayerMS::Load(LoadType load_type, | 234 void WebMediaPlayerMS::Load(LoadType load_type, |
219 const blink::WebMediaPlayerSource& source, | 235 const blink::WebMediaPlayerSource& source, |
220 CORSMode /*cors_mode*/) { | 236 CORSMode /*cors_mode*/) { |
221 DVLOG(1) << __func__; | 237 DVLOG(1) << __func__; |
222 DCHECK(thread_checker_.CalledOnValidThread()); | 238 DCHECK(thread_checker_.CalledOnValidThread()); |
223 | 239 |
224 // TODO(acolwell): Change this to DCHECK_EQ(load_type, LoadTypeMediaStream) | 240 // TODO(acolwell): Change this to DCHECK_EQ(load_type, LoadTypeMediaStream) |
225 // once Blink-side changes land. | 241 // once Blink-side changes land. |
226 DCHECK_NE(load_type, kLoadTypeMediaSource); | 242 DCHECK_NE(load_type, kLoadTypeMediaSource); |
227 blink::WebMediaStream web_stream = | 243 web_stream_ = GetWebMediaStreamFromWebMediaPlayerSource(source); |
228 GetWebMediaStreamFromWebMediaPlayerSource(source); | 244 if (!web_stream_.IsNull()) { |
| 245 MediaStream* native_stream = MediaStream::GetMediaStream(web_stream_); |
| 246 if (native_stream) |
| 247 native_stream->AddObserver(this); |
| 248 } |
229 | 249 |
230 compositor_ = new WebMediaPlayerMSCompositor( | 250 compositor_ = new WebMediaPlayerMSCompositor( |
231 compositor_task_runner_, io_task_runner_, web_stream, AsWeakPtr()); | 251 compositor_task_runner_, io_task_runner_, web_stream_, AsWeakPtr()); |
232 | 252 |
233 SetNetworkState(WebMediaPlayer::kNetworkStateLoading); | 253 SetNetworkState(WebMediaPlayer::kNetworkStateLoading); |
234 SetReadyState(WebMediaPlayer::kReadyStateHaveNothing); | 254 SetReadyState(WebMediaPlayer::kReadyStateHaveNothing); |
235 std::string stream_id = | 255 std::string stream_id = |
236 web_stream.IsNull() ? std::string() : web_stream.Id().Utf8(); | 256 web_stream_.IsNull() ? std::string() : web_stream_.Id().Utf8(); |
237 media_log_->AddEvent(media_log_->CreateLoadEvent(stream_id)); | 257 media_log_->AddEvent(media_log_->CreateLoadEvent(stream_id)); |
238 | 258 |
239 frame_deliverer_.reset(new WebMediaPlayerMS::FrameDeliverer( | 259 frame_deliverer_.reset(new WebMediaPlayerMS::FrameDeliverer( |
240 AsWeakPtr(), | 260 AsWeakPtr(), |
241 base::Bind(&WebMediaPlayerMSCompositor::EnqueueFrame, compositor_))); | 261 base::Bind(&WebMediaPlayerMSCompositor::EnqueueFrame, compositor_))); |
242 video_frame_provider_ = renderer_factory_->GetVideoRenderer( | 262 video_frame_provider_ = renderer_factory_->GetVideoRenderer( |
243 web_stream, media::BindToCurrentLoop(base::Bind( | 263 web_stream_, |
244 &WebMediaPlayerMS::OnSourceError, AsWeakPtr())), | 264 media::BindToCurrentLoop( |
| 265 base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr())), |
245 frame_deliverer_->GetRepaintCallback(), io_task_runner_, | 266 frame_deliverer_->GetRepaintCallback(), io_task_runner_, |
246 media_task_runner_, worker_task_runner_, gpu_factories_); | 267 media_task_runner_, worker_task_runner_, gpu_factories_); |
247 | 268 |
248 RenderFrame* const frame = RenderFrame::FromWebFrame(frame_); | 269 RenderFrame* const frame = RenderFrame::FromWebFrame(frame_); |
249 | 270 |
250 int routing_id = MSG_ROUTING_NONE; | 271 int routing_id = MSG_ROUTING_NONE; |
251 GURL url = source.IsURL() ? GURL(source.GetAsURL()) : GURL(); | 272 GURL url = source.IsURL() ? GURL(source.GetAsURL()) : GURL(); |
252 | 273 |
253 if (frame) { | 274 if (frame) { |
254 // Report UMA and RAPPOR metrics. | 275 // Report UMA and RAPPOR metrics. |
255 media::ReportMetrics(load_type, url, frame_->GetSecurityOrigin(), | 276 media::ReportMetrics(load_type, url, frame_->GetSecurityOrigin(), |
256 media_log_.get()); | 277 media_log_.get()); |
257 routing_id = frame->GetRoutingID(); | 278 routing_id = frame->GetRoutingID(); |
258 } | 279 } |
259 | 280 |
260 audio_renderer_ = renderer_factory_->GetAudioRenderer( | 281 audio_renderer_ = renderer_factory_->GetAudioRenderer( |
261 web_stream, routing_id, initial_audio_output_device_id_, | 282 web_stream_, routing_id, initial_audio_output_device_id_, |
262 initial_security_origin_); | 283 initial_security_origin_); |
263 | 284 |
264 if (!audio_renderer_) | 285 if (!audio_renderer_) |
265 WebRtcLogMessage("Warning: Failed to instantiate audio renderer."); | 286 WebRtcLogMessage("Warning: Failed to instantiate audio renderer."); |
266 | 287 |
267 if (!video_frame_provider_ && !audio_renderer_) { | 288 if (!video_frame_provider_ && !audio_renderer_) { |
268 SetNetworkState(WebMediaPlayer::kNetworkStateNetworkError); | 289 SetNetworkState(WebMediaPlayer::kNetworkStateNetworkError); |
269 return; | 290 return; |
270 } | 291 } |
271 | 292 |
272 if (audio_renderer_) { | 293 if (audio_renderer_) { |
273 audio_renderer_->SetVolume(volume_); | 294 audio_renderer_->SetVolume(volume_); |
274 audio_renderer_->Start(); | 295 audio_renderer_->Start(); |
| 296 |
| 297 // Store the ID of audio track being played in |current_video_track_id_| |
| 298 blink::WebVector<blink::WebMediaStreamTrack> audio_tracks; |
| 299 if (!web_stream_.IsNull()) { |
| 300 web_stream_.AudioTracks(audio_tracks); |
| 301 DCHECK_GT(audio_tracks.size(), 0U); |
| 302 current_audio_track_id_ = audio_tracks[0].Id(); |
| 303 } |
275 } | 304 } |
276 if (video_frame_provider_) | 305 |
| 306 if (video_frame_provider_) { |
277 video_frame_provider_->Start(); | 307 video_frame_provider_->Start(); |
278 | 308 |
| 309 // Store the ID of video track being played in |current_video_track_id_| |
| 310 if (!web_stream_.IsNull()) { |
| 311 blink::WebVector<blink::WebMediaStreamTrack> video_tracks; |
| 312 web_stream_.VideoTracks(video_tracks); |
| 313 DCHECK_GT(video_tracks.size(), 0U); |
| 314 current_video_track_id_ = video_tracks[0].Id(); |
| 315 } |
| 316 } |
279 // When associated with an <audio> element, we don't want to wait for the | 317 // When associated with an <audio> element, we don't want to wait for the |
280 // first video fram to become available as we do for <video> elements | 318 // first video fram to become available as we do for <video> elements |
281 // (<audio> elements can also be assigned video tracks). | 319 // (<audio> elements can also be assigned video tracks). |
282 // For more details, see crbug.com/738379 | 320 // For more details, see crbug.com/738379 |
283 if (audio_renderer_ && | 321 if (audio_renderer_ && |
284 (client_->IsAudioElement() || !video_frame_provider_)) { | 322 (client_->IsAudioElement() || !video_frame_provider_)) { |
285 // This is audio-only mode. | 323 // This is audio-only mode. |
286 SetReadyState(WebMediaPlayer::kReadyStateHaveMetadata); | 324 SetReadyState(WebMediaPlayer::kReadyStateHaveMetadata); |
287 SetReadyState(WebMediaPlayer::kReadyStateHaveEnoughData); | 325 SetReadyState(WebMediaPlayer::kReadyStateHaveEnoughData); |
288 } | 326 } |
289 } | 327 } |
290 | 328 |
| 329 void WebMediaPlayerMS::TrackAdded(const blink::WebMediaStreamTrack& track) { |
| 330 Reload(); |
| 331 } |
| 332 |
| 333 void WebMediaPlayerMS::TrackRemoved(const blink::WebMediaStreamTrack& track) { |
| 334 Reload(); |
| 335 } |
| 336 |
| 337 void WebMediaPlayerMS::Reload() { |
| 338 DCHECK(thread_checker_.CalledOnValidThread()); |
| 339 if (web_stream_.IsNull()) |
| 340 return; |
| 341 |
| 342 ReloadVideo(); |
| 343 ReloadAudio(); |
| 344 } |
| 345 |
| 346 void WebMediaPlayerMS::ReloadVideo() { |
| 347 DCHECK(thread_checker_.CalledOnValidThread()); |
| 348 DCHECK(!web_stream_.IsNull()); |
| 349 blink::WebVector<blink::WebMediaStreamTrack> video_tracks; |
| 350 // VideoTracks() is a getter. |
| 351 web_stream_.VideoTracks(video_tracks); |
| 352 |
| 353 RendererReloadAction renderer_action = RendererReloadAction::KEEP_RENDERER; |
| 354 if (video_tracks.IsEmpty()) { |
| 355 if (video_frame_provider_) |
| 356 renderer_action = RendererReloadAction::REMOVE_RENDERER; |
| 357 current_video_track_id_ = blink::WebString(); |
| 358 } else if (video_tracks[0].Id() != current_video_track_id_) { |
| 359 renderer_action = RendererReloadAction::NEW_RENDERER; |
| 360 current_video_track_id_ = video_tracks[0].Id(); |
| 361 } |
| 362 |
| 363 switch (renderer_action) { |
| 364 case RendererReloadAction::NEW_RENDERER: |
| 365 if (video_frame_provider_) |
| 366 video_frame_provider_->Stop(); |
| 367 |
| 368 video_frame_provider_ = renderer_factory_->GetVideoRenderer( |
| 369 web_stream_, |
| 370 media::BindToCurrentLoop( |
| 371 base::Bind(&WebMediaPlayerMS::OnSourceError, AsWeakPtr())), |
| 372 frame_deliverer_->GetRepaintCallback(), io_task_runner_, |
| 373 media_task_runner_, worker_task_runner_, gpu_factories_); |
| 374 DCHECK(video_frame_provider_); |
| 375 video_frame_provider_->Start(); |
| 376 break; |
| 377 |
| 378 case RendererReloadAction::REMOVE_RENDERER: |
| 379 video_frame_provider_->Stop(); |
| 380 video_frame_provider_ = nullptr; |
| 381 break; |
| 382 |
| 383 default: |
| 384 return; |
| 385 } |
| 386 |
| 387 DCHECK_NE(renderer_action, RendererReloadAction::KEEP_RENDERER); |
| 388 if (!paused_) |
| 389 delegate_->DidPlayerSizeChange(delegate_id_, NaturalSize()); |
| 390 } |
| 391 |
| 392 void WebMediaPlayerMS::ReloadAudio() { |
| 393 DCHECK(thread_checker_.CalledOnValidThread()); |
| 394 DCHECK(!web_stream_.IsNull()); |
| 395 RenderFrame* const frame = RenderFrame::FromWebFrame(frame_); |
| 396 if (!frame) |
| 397 return; |
| 398 |
| 399 blink::WebVector<blink::WebMediaStreamTrack> audio_tracks; |
| 400 // AudioTracks() is a getter. |
| 401 web_stream_.AudioTracks(audio_tracks); |
| 402 |
| 403 RendererReloadAction renderer_action = RendererReloadAction::KEEP_RENDERER; |
| 404 if (audio_tracks.IsEmpty()) { |
| 405 if (audio_renderer_) |
| 406 renderer_action = RendererReloadAction::REMOVE_RENDERER; |
| 407 current_audio_track_id_ = blink::WebString(); |
| 408 } else if (audio_tracks[0].Id() != current_video_track_id_) { |
| 409 renderer_action = RendererReloadAction::NEW_RENDERER; |
| 410 current_audio_track_id_ = audio_tracks[0].Id(); |
| 411 } |
| 412 |
| 413 switch (renderer_action) { |
| 414 case RendererReloadAction::NEW_RENDERER: |
| 415 if (audio_renderer_) |
| 416 audio_renderer_->Stop(); |
| 417 |
| 418 audio_renderer_ = renderer_factory_->GetAudioRenderer( |
| 419 web_stream_, frame->GetRoutingID(), initial_audio_output_device_id_, |
| 420 initial_security_origin_); |
| 421 audio_renderer_->SetVolume(volume_); |
| 422 audio_renderer_->Start(); |
| 423 audio_renderer_->Play(); |
| 424 break; |
| 425 |
| 426 case RendererReloadAction::REMOVE_RENDERER: |
| 427 audio_renderer_->Stop(); |
| 428 audio_renderer_ = nullptr; |
| 429 break; |
| 430 |
| 431 default: |
| 432 break; |
| 433 } |
| 434 } |
| 435 |
291 void WebMediaPlayerMS::Play() { | 436 void WebMediaPlayerMS::Play() { |
292 DVLOG(1) << __func__; | 437 DVLOG(1) << __func__; |
293 DCHECK(thread_checker_.CalledOnValidThread()); | 438 DCHECK(thread_checker_.CalledOnValidThread()); |
294 | 439 |
295 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY)); | 440 media_log_->AddEvent(media_log_->CreateEvent(media::MediaLogEvent::PLAY)); |
296 if (!paused_) | 441 if (!paused_) |
297 return; | 442 return; |
298 | 443 |
299 if (video_frame_provider_) | 444 if (video_frame_provider_) |
300 video_frame_provider_->Resume(); | 445 video_frame_provider_->Resume(); |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
387 return (video_frame_provider_.get() != nullptr); | 532 return (video_frame_provider_.get() != nullptr); |
388 } | 533 } |
389 | 534 |
390 bool WebMediaPlayerMS::HasAudio() const { | 535 bool WebMediaPlayerMS::HasAudio() const { |
391 DCHECK(thread_checker_.CalledOnValidThread()); | 536 DCHECK(thread_checker_.CalledOnValidThread()); |
392 return (audio_renderer_.get() != nullptr); | 537 return (audio_renderer_.get() != nullptr); |
393 } | 538 } |
394 | 539 |
395 blink::WebSize WebMediaPlayerMS::NaturalSize() const { | 540 blink::WebSize WebMediaPlayerMS::NaturalSize() const { |
396 DCHECK(thread_checker_.CalledOnValidThread()); | 541 DCHECK(thread_checker_.CalledOnValidThread()); |
| 542 if (!video_frame_provider_) |
| 543 return blink::WebSize(); |
| 544 |
397 if (video_rotation_ == media::VIDEO_ROTATION_90 || | 545 if (video_rotation_ == media::VIDEO_ROTATION_90 || |
398 video_rotation_ == media::VideoRotation::VIDEO_ROTATION_270) { | 546 video_rotation_ == media::VideoRotation::VIDEO_ROTATION_270) { |
399 const gfx::Size& current_size = compositor_->GetCurrentSize(); | 547 const gfx::Size& current_size = compositor_->GetCurrentSize(); |
400 return blink::WebSize(current_size.height(), current_size.width()); | 548 return blink::WebSize(current_size.height(), current_size.width()); |
401 } | 549 } |
402 return blink::WebSize(compositor_->GetCurrentSize()); | 550 return blink::WebSize(compositor_->GetCurrentSize()); |
403 } | 551 } |
404 | 552 |
405 blink::WebSize WebMediaPlayerMS::VisibleRect() const { | 553 blink::WebSize WebMediaPlayerMS::VisibleRect() const { |
406 DCHECK(thread_checker_.CalledOnValidThread()); | 554 DCHECK(thread_checker_.CalledOnValidThread()); |
(...skipping 342 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
749 video_renderer_.ResetCache(); | 897 video_renderer_.ResetCache(); |
750 } | 898 } |
751 | 899 |
752 void WebMediaPlayerMS::TriggerResize() { | 900 void WebMediaPlayerMS::TriggerResize() { |
753 get_client()->SizeChanged(); | 901 get_client()->SizeChanged(); |
754 | 902 |
755 delegate_->DidPlayerSizeChange(delegate_id_, NaturalSize()); | 903 delegate_->DidPlayerSizeChange(delegate_id_, NaturalSize()); |
756 } | 904 } |
757 | 905 |
758 } // namespace content | 906 } // namespace content |
OLD | NEW |