[Video] MediaSession API event handlers can resume background video.

media/blink/webmediaplayer_impl.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/blink/webmediaplayer_impl.h"
 
 #include <algorithm>
 #include <cmath>
 #include <limits>
 #include <string>
@@ skipping 45 matching lines @@
 #include "third_party/WebKit/public/platform/WebMediaPlayerSource.h"
 #include "third_party/WebKit/public/platform/WebMediaSource.h"
 #include "third_party/WebKit/public/platform/WebRect.h"
 #include "third_party/WebKit/public/platform/WebSecurityOrigin.h"
 #include "third_party/WebKit/public/platform/WebSize.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 #include "third_party/WebKit/public/platform/WebURL.h"
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebFrame.h"
 #include "third_party/WebKit/public/web/WebLocalFrame.h"
+#include "third_party/WebKit/public/web/WebUserGestureIndicator.h"
 #include "third_party/WebKit/public/web/WebView.h"
 
 #if defined(OS_ANDROID)
 #include "media/base/android/media_codec_util.h"
 #endif
 
 using blink::WebCanvas;
 using blink::WebMediaPlayer;
 using blink::WebRect;
 using blink::WebSize;
@@ skipping 335 matching lines @@
 
 #if defined(OS_ANDROID)  // WMPI_CAST
   cast_impl_.Initialize(url, frame_, delegate_id_);
 #endif
 }
 
 void WebMediaPlayerImpl::play() {
   DVLOG(1) << __func__;
   DCHECK(main_task_runner_->BelongsToCurrentThread());
 
+  // User initiated play unlocks background video playback.
+  if (blink::WebUserGestureIndicator::isProcessingUserGesture())

[sandersd (OOO until July 31), 2017/02/23 00:38:31]

Should we be consuming the gesture?

[whywhat, 2017/02/23 02:50:27]

I don't think so, the page may call play() and pause() in an event handler as
many times as it wants. IMO it'd be unexpected to have the first play() work and
the rest stop working.

+    video_locked_when_paused_when_hidden_ = false;
+
 #if defined(OS_ANDROID)  // WMPI_CAST
   if (isRemote()) {
     cast_impl_.play();
     return;
   }
 #endif
   // TODO(sandersd): Do we want to reset the idle timer here?
   delegate_->SetIdle(delegate_id_, false);
   paused_ = false;
   pipeline_.SetPlaybackRate(playback_rate_);
@@ skipping 16 matching lines @@
   DCHECK(main_task_runner_->BelongsToCurrentThread());
 
   // We update the paused state even when casting, since we expect pause() to be
   // called when casting begins, and when we exit casting we should end up in a
   // paused state.
   paused_ = true;
 
   // No longer paused because it was hidden.
   paused_when_hidden_ = false;
 
+  // User initiated pause locks background videos.
+  if (blink::WebUserGestureIndicator::isProcessingUserGesture())
+    video_locked_when_paused_when_hidden_ = true;
+
 #if defined(OS_ANDROID)  // WMPI_CAST
   if (isRemote()) {
     cast_impl_.pause();
     return;
   }
 #endif
 
   pipeline_.SetPlaybackRate(0.0);
 
   // pause() may be called after playback has ended and the HTMLMediaElement
@@ skipping 960 matching lines @@
     video_weblayer_->layer()->SetContentsOpaque(opaque_);
 }
 
 void WebMediaPlayerImpl::OnVideoAverageKeyframeDistanceUpdate() {
   UpdateBackgroundVideoOptimizationState();
 }
 
 void WebMediaPlayerImpl::OnFrameHidden() {
   DCHECK(main_task_runner_->BelongsToCurrentThread());
 
+  // Backgrounding a video requires a user gesture to resume playback.
+  if (IsHidden())
+    video_locked_when_paused_when_hidden_ = true;
+
   if (watch_time_reporter_)
     watch_time_reporter_->OnHidden();
 
-  // OnFrameHidden() can be called when frame is closed, then IsHidden() will
-  // return false, so check explicitly.
-  if (IsHidden()) {
-    if (ShouldPauseVideoWhenHidden()) {
-      PauseVideoIfNeeded();
-      return;
-    } else {
-      DisableVideoTrackIfNeeded();
-    }
-  }
-
+  UpdateBackgroundVideoOptimizationState();
   UpdatePlayState();
 
   // Schedule suspended playing media to be paused if the user doesn't come back
   // to it within some timeout period to avoid any autoplay surprises.
   ScheduleIdlePauseTimer();
 }
 
 void WebMediaPlayerImpl::OnFrameClosed() {
   DCHECK(main_task_runner_->BelongsToCurrentThread());
   UpdatePlayState();
 }
 
 void WebMediaPlayerImpl::OnFrameShown() {
   DCHECK(main_task_runner_->BelongsToCurrentThread());
   background_pause_timer_.Stop();
 
+  // Foreground videos don't require user gesture to continue playback.
+  video_locked_when_paused_when_hidden_ = false;
+
   if (watch_time_reporter_)
     watch_time_reporter_->OnShown();
 
   // Only track the time to the first frame if playing or about to play because
   // of being shown and only for videos we would optimize background playback
   // for.
   if ((!paused_ && IsBackgroundOptimizationCandidate()) ||
       paused_when_hidden_) {
     VideoFrameCompositor::OnNewProcessedFrameCB new_processed_frame_cb =
         BindToCurrentLoop(base::Bind(
             &WebMediaPlayerImpl::ReportTimeFromForegroundToFirstFrame,
             AsWeakPtr(), base::TimeTicks::Now()));
     compositor_task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&VideoFrameCompositor::SetOnNewProcessedFrameCallback,
                    base::Unretained(compositor_), new_processed_frame_cb));
   }
 
+  EnableVideoTrackIfNeeded();
+
   if (paused_when_hidden_) {
     paused_when_hidden_ = false;
     OnPlay();  // Calls UpdatePlayState() so return afterwards.
     return;
   }
 
-  EnableVideoTrackIfNeeded();
-
   UpdatePlayState();
 }
 
 void WebMediaPlayerImpl::OnIdleTimeout() {
   DCHECK(main_task_runner_->BelongsToCurrentThread());
 
   // If we are attempting preroll, clear the stale flag.
   if (IsPrerollAttemptNeeded()) {
     delegate_->ClearStaleFlag(delegate_id_);
     return;
@@ skipping 444 matching lines @@
                                                      bool is_backgrounded) {
   PlayState result;
 
   bool must_suspend = delegate_->IsFrameClosed();
   bool is_stale = delegate_->IsStale(delegate_id_);
 
   // This includes both data source (before pipeline startup) and pipeline
   // errors.
   bool has_error = IsNetworkStateError(network_state_);
 
-  // After HaveMetadata, we know which tracks are present and the duration.
-  bool have_metadata = ready_state_ >= WebMediaPlayer::ReadyStateHaveMetadata;
-
   // After HaveFutureData, Blink will call play() if the state is not paused;
   // prior to this point |paused_| is not accurate.
   bool have_future_data =
       highest_ready_state_ >= WebMediaPlayer::ReadyStateHaveFutureData;
 
-  // Background suspend is not enabled for audio-only players unless paused,
-  // though in the case of audio-only the session should be kept.
-  // Videos are not suspended if the user resumed the playback via the remote
-  // controls earlier and it's still playing.
-  bool is_backgrounded_video = is_backgrounded && have_metadata && hasVideo();
-  bool can_play_backgrounded = is_backgrounded_video && !is_remote &&
-                               hasAudio() && IsResumeBackgroundVideosEnabled();
-  bool is_background_playing = delegate_->IsBackgroundVideoPlaybackUnlocked();
-  bool background_suspended = can_auto_suspend && is_backgrounded_video &&
-                              !(can_play_backgrounded && is_background_playing);
-  bool background_pause_suspended =
+  // Background suspend is only enabled for paused players.
+  // In the case of players with audio the session should be kept.
+  bool background_suspended =
       can_auto_suspend && is_backgrounded && paused_ && have_future_data;
 
   // Idle suspension is allowed prior to have future data since there exist
   // mechanisms to exit the idle state when the player is capable of reaching
   // the have future data state; see didLoadingProgress().
   //
   // TODO(sandersd): Make the delegate suspend idle players immediately when
   // hidden.
   bool idle_suspended =
       can_auto_suspend && is_stale && paused_ && !seeking_ && !overlay_enabled_;
 
   // If we're already suspended, see if we can wait for user interaction. Prior
   // to HaveFutureData, we require |is_stale| to remain suspended. |is_stale|
   // will be cleared when we receive data which may take us to HaveFutureData.
   bool can_stay_suspended =
       (is_stale || have_future_data) && is_suspended && paused_ && !seeking_;
 
   // Combined suspend state.
   result.is_suspended = is_remote || must_suspend || idle_suspended ||
-                        background_suspended || background_pause_suspended ||
-                        can_stay_suspended;
+                        background_suspended || can_stay_suspended;
 
   // We do not treat |playback_rate_| == 0 as paused. For the media session,
   // being paused implies displaying a play button, which is incorrect in this
   // case. For memory usage reporting, we just use the same definition (but we
   // don't have to).
   //
   // Similarly, we don't consider |ended_| to be paused. Blink will immediately
   // call pause() or seek(), so |ended_| should not affect the computation.
   // Despite that, |ended_| does result in a separate paused state, to simplfy
   // the contract for SetDelegateState().
   //
   // |has_session| is used to decide when to create a media session. Idle
   // suspension does not destroy the media session, because we expect that the
   // notification controls (and audio focus) remain. We also require:
   //   - |have_metadata|, since the tracks and duration are passed to DidPlay().
   //   - |have_future_data|, since we need to know whether we are paused to
   //     correctly configure the session.
   //
   // TODO(sandersd): If Blink told us the paused state sooner, we could create
   // the media session sooner.
   bool can_play = !has_error && !is_remote && have_future_data;
-  bool has_session_playing = can_play && !must_suspend && !background_suspended;
-
-  // |has_session_suspended| means the player is suspended from the media
-  // element point of view but paused and can be resumed from the delegate point
-  // of view. Therefore it behaves like |paused_| for the delegate.
-  bool has_session_suspended = can_play && !must_suspend &&
-                               background_suspended && can_play_backgrounded;
-
-  bool has_session = has_session_playing || has_session_suspended;
+  bool has_session =
+      can_play && !must_suspend && hasAudio() &&
+      (!hasVideo() || !is_backgrounded || !IsBackgroundedSuspendEnabled() ||

[sandersd (OOO until July 31), 2017/02/23 00:38:31]

This last condition isn't obvious, please pull it out into a variable and
comment it.

Also please make sure the large block comment is updated; it's not obvious how
eg. |have_metadata| is relevant when that variable doesn't exist anymore. (But
you are depending on it here--|!hasVideo()| is always true before
|have_metadata|.)

[whywhat, 2017/02/23 02:50:27]

Done. |have_metadata| is implied by |has_future_data| so I removed it. Also
renamed has_session to has_remote_controls as session is now also refers to the
API.

+       IsResumeBackgroundVideosEnabled());
 
   if (!has_session) {
     result.delegate_state = DelegateState::GONE;
     result.is_idle = delegate_->IsIdle(delegate_id_);
-  } else if (paused_ || has_session_suspended) {
+  } else if (paused_) {
     // TODO(sandersd): Is it possible to have a suspended session, be ended,
     // and not be paused? If so we should be in a PLAYING state.
     result.delegate_state =
         ended_ ? DelegateState::GONE : DelegateState::PAUSED;
     result.is_idle = !seeking_;
   } else {
     result.delegate_state = DelegateState::PLAYING;
     result.is_idle = false;
   }
 
@@ skipping 57 matching lines @@
     UMA_HISTOGRAM_MEMORY_KB("Media.WebMediaPlayerImpl.Memory.DataSource",
                             data_source_memory_usage / 1024);
   }
   if (demuxer_) {
     UMA_HISTOGRAM_MEMORY_KB("Media.WebMediaPlayerImpl.Memory.Demuxer",
                             demuxer_memory_usage / 1024);
   }
 }
 
 void WebMediaPlayerImpl::ScheduleIdlePauseTimer() {
-  // Only schedule the pause timer if we're playing and are suspended.
-  if (paused_ || !pipeline_controller_.IsSuspended())
+  // Only schedule the pause timer if we're not paused or paused but going to
+  // resume when foregrounded, and are suspended.
+  if ((paused_ && !paused_when_hidden_) || !pipeline_controller_.IsSuspended())
     return;
 
 #if defined(OS_ANDROID)
   // Remote players will be suspended and locally paused.
   if (isRemote())
     return;
 #endif
 
   // Idle timeout chosen arbitrarily.
   background_pause_timer_.Start(FROM_HERE, base::TimeDelta::FromSeconds(5),
@@ skipping 27 matching lines @@
   return pipeline_metadata_.video_rotation == VIDEO_ROTATION_0;
 }
 
 void WebMediaPlayerImpl::ActivateViewportIntersectionMonitoring(bool activate) {
   DCHECK(main_task_runner_->BelongsToCurrentThread());
 
   client_->activateViewportIntersectionMonitoring(activate);
 }
 
 bool WebMediaPlayerImpl::ShouldPauseVideoWhenHidden() const {
-#if !defined(OS_ANDROID)
-  // On desktop, this behavior is behind the feature flag.
-  if (!IsBackgroundVideoTrackOptimizationEnabled())
-    return false;
+  // If suspending background video, pause any video that's not remoted or
+  // not unlocked to play in the background.
+  if (IsBackgroundedSuspendEnabled()) {
+    if (!hasVideo())
+      return false;
+
+#if defined(OS_ANDROID)
+    if (isRemote())
+      return false;
 #endif
 
-  // Pause video-only players that match the criteria for being optimized.
-  return !hasAudio() && IsBackgroundOptimizationCandidate();
+    return !hasAudio() ||
+           (IsResumeBackgroundVideosEnabled() &&
+            video_locked_when_paused_when_hidden_);
+  }
+
+  // Otherwise only pause if the optimization is on and it's a video-only
+  // optimization candidate.
+  return IsBackgroundVideoTrackOptimizationEnabled() && !hasAudio() &&
+         IsBackgroundOptimizationCandidate();
 }
 
 bool WebMediaPlayerImpl::ShouldDisableVideoWhenHidden() const {
   // This optimization is behind the flag on all platforms.
   if (!IsBackgroundVideoTrackOptimizationEnabled())
     return false;
 
   // Disable video track only for players with audio that match the criteria for
   // being optimized.
   return hasAudio() && IsBackgroundOptimizationCandidate();
@@ skipping 110 matching lines @@
   base::TimeDelta time_to_first_frame = new_frame_time - foreground_time;
   if (hasAudio()) {
     UMA_HISTOGRAM_TIMES(
         "Media.Video.TimeFromForegroundToFirstFrame.DisableTrack",
         time_to_first_frame);
   } else {
     UMA_HISTOGRAM_TIMES("Media.Video.TimeFromForegroundToFirstFrame.Paused",
                         time_to_first_frame);
   }
 }
-
 void WebMediaPlayerImpl::SwitchRenderer(bool disable_pipeline_auto_suspend) {
   DCHECK(main_task_runner_->BelongsToCurrentThread());
   disable_pipeline_auto_suspend_ = disable_pipeline_auto_suspend;
   ScheduleRestart();
 }
 
 }  // namespace media