Implement autoplay gesture override experiment.

Source/core/html/HTMLMediaElement.cpp

 /*
  * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 108 matching lines @@
 
 #ifndef LOG_CACHED_TIME_WARNINGS
 // Default to not logging warnings about excessive drift in the cached media time because it adds a
 // fair amount of overhead and logging.
 #define LOG_CACHED_TIME_WARNINGS 0
 #endif
 
 // URL protocol used to signal that the media source API is being used.
 static const char mediaSourceBlobProtocol[] = "blob";
 
+// How often do we poll for scrolling stopped during a visibility check?
+static const double visibilityTimerPollDelay = 0.5;
+// How long do we repeat visibility checks?  We will poll once every PollDelay.
+// Note that we will stop checking if we don't detect scrolling, also.
+static const double visibilityCheckDuration = 5;
+
 using namespace HTMLNames;
 
 typedef WillBeHeapHashSet<RawPtrWillBeWeakMember<HTMLMediaElement>> WeakMediaElementSet;
 typedef WillBeHeapHashMap<RawPtrWillBeWeakMember<Document>, WeakMediaElementSet> DocumentElementSetMap;
 static DocumentElementSetMap& documentToElementSetMap()
 {
     DEFINE_STATIC_LOCAL(OwnPtrWillBePersistent<DocumentElementSetMap>, map, (adoptPtrWillBeNoop(new DocumentElementSetMap())));
     return *map;
 }
 
@@ skipping 105 matching lines @@
     // when used with parameters, e.g. "application/octet-stream;codecs=theora", is a type that the user agent knows
     // it cannot render.
     if (contentMIMEType != "application/octet-stream" || contentTypeCodecs.isEmpty()) {
         WebMimeRegistry::SupportsType supported = Platform::current()->mimeRegistry()->supportsMediaMIMEType(contentMIMEType, contentTypeCodecs, keySystem.lower());
         return supported > WebMimeRegistry::IsNotSupported;
     }
 
     return false;
 }
 
-// These values are used for a histogram. Do not reorder.
-enum AutoplayMetrics {
-    // Media element with autoplay seen.
-    AutoplayMediaFound = 0,
-    // Autoplay enabled and user stopped media play at any point.
-    AutoplayStopped = 1,
-    // Autoplay enabled but user bailed out on media play early.
-    AutoplayBailout = 2,
-    // Autoplay disabled but user manually started media.
-    AutoplayManualStart = 3,
-    // Autoplay was (re)enabled through a user-gesture triggered load()
-    AutoplayEnabledThroughLoad = 4,
-    // Autoplay disabled by sandbox flags.
-    AutoplayDisabledBySandbox = 5,
-    // This enum value must be last.
-    NumberOfAutoplayMetrics,
-};
-
-static void recordAutoplayMetric(AutoplayMetrics metric)
+void HTMLMediaElement::recordAutoplayMetric(AutoplayMetrics metric)
 {
     Platform::current()->histogramEnumeration("Blink.MediaElement.Autoplay", metric, NumberOfAutoplayMetrics);
 }
 
 WebMimeRegistry::SupportsType HTMLMediaElement::supportsType(const ContentType& contentType, const String& keySystem)
 {
     DEFINE_STATIC_LOCAL(const String, codecs, ("codecs"));
 
     if (!RuntimeEnabledFeatures::mediaEnabled())
         return WebMimeRegistry::IsNotSupported;
@@ skipping 10 matching lines @@
     // 4.8.10.3 MIME types - The canPlayType(type) method must return the empty string if type is a type that the
     // user agent knows it cannot render or is the type "application/octet-stream"
     if (type == "application/octet-stream")
         return WebMimeRegistry::IsNotSupported;
 
     return Platform::current()->mimeRegistry()->supportsMediaMIMEType(type, typeCodecs, system);
 }
 
 URLRegistry* HTMLMediaElement::s_mediaStreamRegistry = 0;
 
+class HTMLMediaElement::AutoplayExperimentTouchListener : public EventListener  {
+    public:
+        AutoplayExperimentTouchListener(HTMLMediaElement* element) : EventListener(CPPEventListenerType), m_element(element) { }
+        virtual bool operator==(const EventListener& them)
+        {
+            return &them == this;
+        }
+
+        void handleEvent(ExecutionContext*, Event*) override
+        {
+            if (m_element)
+                m_element->beginPeriodicVisibilityCheck();
+        }
+
+    private:
+        HTMLMediaElement* m_element;
+};
+
 void HTMLMediaElement::setMediaStreamRegistry(URLRegistry* registry)
 {
     ASSERT(!s_mediaStreamRegistry);
     s_mediaStreamRegistry = registry;
 }
 
 bool HTMLMediaElement::isMediaStreamURL(const String& url)
 {
     return s_mediaStreamRegistry ? s_mediaStreamRegistry->contains(url) : false;
 }
@@ skipping 44 matching lines @@
     , m_haveVisibleTextTrack(false)
     , m_processingPreferenceChange(false)
     , m_remoteRoutesAvailable(false)
     , m_playingRemotely(false)
     , m_isFinalizing(false)
     , m_initialPlayWithoutUserGestures(false)
     , m_autoplayMediaCounted(false)
     , m_audioTracks(AudioTrackList::create(*this))
     , m_videoTracks(VideoTrackList::create(*this))
     , m_textTracks(nullptr)
+    , m_autoplayExperimentPlayPending(false)
+    , m_autoplayExperimentStartedByExperiment(false)
+    , m_autoplayExperimentMode(ExperimentOff)
 #if ENABLE(WEB_AUDIO)
     , m_audioSourceNode(nullptr)
 #endif
+    , m_autoplayVisibilityTimer(this, &HTMLMediaElement::visibilityTimerFired)
+    , m_autoplayLastScrollX(std::numeric_limits<double>::quiet_NaN())
+    , m_autoplayLastScrollY(std::numeric_limits<double>::quiet_NaN())
+    , m_autoplayVisibilityTimerSpan(0)
 {
 #if ENABLE(OILPAN)
     ThreadState::current()->registerPreFinalizer(this);
 #endif
     ASSERT(RuntimeEnabledFeatures::mediaEnabled());
 
     WTF_LOG(Media, "HTMLMediaElement::HTMLMediaElement(%p)", this);
 
-    if (document.settings() && document.settings()->mediaPlaybackRequiresUserGesture())

[philipj_slow, 2015/08/05 10:03:11]

Wow, this has actually been broken the whole time, you can already circumvent
the restriction entirely by getting a document with no settings object:

var video = document.querySelector("video");
var doc = document.implementation.createHTMLDocument();
doc.body.appendChild(video.cloneNode(true));
var autoplayVideo = doc.body.firstChild.cloneNode(true);
video.parentNode.replaceChild(autoplayVideo, video);
autoplayVideo.play();

So, yeah, we should probably make all of this not suck so much. Not sure what
would happen if this trick was widely known... o_O

[liberato (no reviews please), 2015/08/06 06:37:58]

i'm glad you're one of the good guys.

-        m_userGestureRequiredForPlay = true;
+    if (document.settings()) {
+        if (document.settings()->mediaPlaybackRequiresUserGesture())
+            m_userGestureRequiredForPlay = true;
+
+        const String& autoplayMode = document.settings()->autoplayExperimentMode();
+        if (autoplayMode.contains("enabled")) {
+            // Autoplay with no gesture requirement.
+            m_autoplayExperimentMode |= ExperimentEnabled;
+        }
+        if (autoplayMode.contains("-ifvisible")) {
+            // Override gesture requirement only if the player is visible.
+            m_autoplayExperimentMode |= ExperimentIfVisible;
+        }
+        if (autoplayMode.contains("-ifmuted")) {
+            // Override gesture requirement only if the media is muted or has
+            // no audio track.
+            m_autoplayExperimentMode |= ExperimentIfMuted;
+        }
+        if (autoplayMode.contains("-ifmobile")) {
+            // Override gesture requirement only if the page is optimized
+            // for mobile.
+            m_autoplayExperimentMode |= ExperimentIfMobile;
+        }
+        if (autoplayMode.contains("-playmuted")) {
+            m_autoplayExperimentMode |= ExperimentPlayMuted;
+        }
+
+        if (m_autoplayExperimentMode != ExperimentOff) {
+            WTF_LOG(Media, "HTMLMediaElement: autoplay experiment set to '%s' (%d)",
+                autoplayMode.ascii().data(), m_autoplayExperimentMode);
+        }
+    }
 
     setHasCustomStyleCallbacks();
     addElementToDocumentMap(this, &document);
 }
 
 HTMLMediaElement::~HTMLMediaElement()
 {
     WTF_LOG(Media, "HTMLMediaElement::~HTMLMediaElement(%p)", this);
+
+    autoplayExperimentClearEventListenerIfNeeded();
+
 #if !ENABLE(OILPAN)
     // HTMLMediaElement and m_asyncEventQueue always become unreachable
     // together. So HTMLMediaElement and m_asyncEventQueue are destructed in
     // the same GC. We don't need to close it explicitly in Oilpan.
     m_asyncEventQueue->close();
 
     setShouldDelayLoadEvent(false);
 
     if (m_textTracks)
         m_textTracks->clearOwner();
@@ skipping 1150 matching lines @@
             scheduleEvent(EventTypeNames::canplay);
             if (isPotentiallyPlaying)
                 scheduleEvent(EventTypeNames::playing);
         }
 
         if (m_autoplaying && m_paused && autoplay()) {
             autoplayMediaEncountered();
 
             if (document().isSandboxed(SandboxAutomaticFeatures)) {
                 recordAutoplayMetric(AutoplayDisabledBySandbox);
-            } else if (!m_userGestureRequiredForPlay) {
-                m_paused = false;
-                invalidateCachedTime();
-                scheduleEvent(EventTypeNames::play);
-                scheduleEvent(EventTypeNames::playing);
+            } else {
+                // If the autoplay experiment says that it's okay to play now,
+                // then don't require a user gesture.
+                if (autoplayExperimentIsEligible()) {
+                    if (autoplayExperimentIsVisible()) {

[philipj_slow, 2015/08/05 10:03:11]

Perhaps rename this so that it's clear that it always returns true when
visibility isn't required, or move that checks to the call sites if that would
be clearer.

[liberato (no reviews please), 2015/08/06 06:37:57]

Done.

+                        autoplayExperimentPrepareToPlay(AutoplayExperimentStartedByLoad);
+                        // Will play below.
+                    } else {
+                        // Wait for visibility checks to pass.
+                        autoplayExperimentInstallEventListenerIfNeeded();
+                    }
+                }
+
+                if (!m_userGestureRequiredForPlay) {
+                    m_paused = false;

[philipj_slow, 2015/08/05 10:03:11]

This branch was previously predicated on the sandbox check passing, was that an
intentional change?

[liberato (no reviews please), 2015/08/06 06:37:57]

it still is, though it's virtually impossible to tell from the formatting here.

+                    invalidateCachedTime();
+                    scheduleEvent(EventTypeNames::play);
+                    scheduleEvent(EventTypeNames::playing);
+                }
             }
         }
 
         scheduleEvent(EventTypeNames::canplaythrough);
 
         shouldUpdateDisplayState = true;
     }
 
     if (shouldUpdateDisplayState) {
         updateDisplayState();
@@ skipping 380 matching lines @@
 
 WebMediaPlayer::Preload HTMLMediaElement::effectivePreloadType() const
 {
     return autoplay() ? WebMediaPlayer::PreloadAuto : preloadType();
 }
 
 void HTMLMediaElement::play()
 {
     WTF_LOG(Media, "HTMLMediaElement::play(%p)", this);
 
+    // Set the pending state, even if the play isn't going to be pending.
+    // Eligibility can change if, for example, the mute status changes.
+    // Having this set is okay.
+    m_autoplayExperimentPlayPending = true;
+
     if (!UserGestureIndicator::processingUserGesture()) {
         autoplayMediaEncountered();
+
+        if (autoplayExperimentIsEligible()) {
+            // Remember that m_userGestureRequiredForPlay is required for
+            // us to be eligible for the experiment.
+            // If we are able to override the gesture requirement now, then
+            // do so.  Otherwise, install an event listener if we need one.
+            if (autoplayExperimentIsVisible()) {
+                // Override the gesture and play.
+                autoplayExperimentPrepareToPlay(AutoplayExperimentStartedByPlay);
+            } else {
+                // Wait for visibility.
+                autoplayExperimentInstallEventListenerIfNeeded();
+            }
+        }
+
         if (m_userGestureRequiredForPlay) {
+            recordAutoplayMetric(AutoplayPlayFailed);
             String message = ExceptionMessages::failedToExecute("play", "HTMLMediaElement", "API can only be initiated by a user gesture.");
             document().executionContext()->addConsoleMessage(ConsoleMessage::create(JSMessageSource, WarningMessageLevel, message));
             return;
         }
     } else if (m_userGestureRequiredForPlay) {
         if (m_autoplayMediaCounted)
             recordAutoplayMetric(AutoplayManualStart);
         m_userGestureRequiredForPlay = false;
+        autoplayExperimentClearEventListenerIfNeeded();
     }
 
     playInternal();
 }
 
 void HTMLMediaElement::playInternal()
 {
     WTF_LOG(Media, "HTMLMediaElement::playInternal(%p)", this);
 
     // 4.8.10.9. Playing the media resource
@@ skipping 41 matching lines @@
     ASSERT(m_initialPlayWithoutUserGestures);
     m_initialPlayWithoutUserGestures = false;
 
     recordAutoplayMetric(AutoplayStopped);
 
     // We count the user as having bailed-out on the video if they watched
     // less than one minute and less than 50% of it.
     double playedTime = currentTime();
     if (playedTime < 60) {
         double progress = playedTime / duration();
-        if (progress < 0.5)
+        if (progress < 0.5) {

[philipj_slow, 2015/08/05 10:03:11]

Why the added {}?

[liberato (no reviews please), 2015/08/06 06:37:57]

whoops, thanks -- i had more code in there and forgot to remove.  i still have a
strong habit for {} that requires conscious effort to override.

             recordAutoplayMetric(AutoplayBailout);
+        }
     }
 }
 
 void HTMLMediaElement::pause()
 {
     WTF_LOG(Media, "HTMLMediaElement::pause(%p)", this);
 
     if (m_networkState == NETWORK_EMPTY)
         scheduleDelayedAction(LoadMediaResource);
 
+    // Don't try to autoplay, if we would have.
+    m_autoplayExperimentPlayPending = false;
+    autoplayExperimentClearEventListenerIfNeeded();
+
     m_autoplaying = false;
 
     if (!m_paused) {
         if (m_initialPlayWithoutUserGestures)
             gesturelessInitialPlayHalted();
 
         m_paused = true;
         scheduleTimeupdateEvent(false);
         scheduleEvent(EventTypeNames::pause);
     }
@@ skipping 77 matching lines @@
 
 void HTMLMediaElement::setMuted(bool muted)
 {
     WTF_LOG(Media, "HTMLMediaElement::setMuted(%p, %s)", this, boolString(muted));
 
     if (m_muted == muted)
         return;
 
     m_muted = muted;
 
+    // If we are no longer eligible for the autoplay experiment, then also
+    // quit listening for events.  If we are eligible, and if we should be
+    // playing, then start playing.  In other words, start playing if

[philipj_slow, 2015/08/05 10:03:11]

Really start playing by setting the muted attribute? Even with no intention of
shipping this, I'm not sure what could be learned from this.

[liberato (no reviews please), 2015/08/06 06:37:57]

there was interest in this from one of the original threads on the subject.

for me, comparing bailout rates of "visible", "visible but no audio" and
"visible and we turned off the audio" might be interesting.  particularly, i'm
curious to know if the latter two bailout rates are different.  it won't be
enough to figure out why, but it will tell us if there's something to look
deeper at.

[philipj_slow, 2015/08/13 09:27:53]

Acknowledged.

+    // we just needed 'mute' to autoplay.
+    if (!autoplayExperimentIsEligible()) {
+        autoplayExperimentClearEventListenerIfNeeded();
+    } else {
+        // Try to play.  If we can't, then install a visibility listener.
+        if (!autoplayExperimentMaybeStartPlaying())
+            autoplayExperimentInstallEventListenerIfNeeded();
+    }
+
     updateVolume();
 
     scheduleEvent(EventTypeNames::volumechange);
 }
 
 void HTMLMediaElement::updateVolume()
 {
     if (webMediaPlayer())
         webMediaPlayer()->setVolume(effectiveMediaVolume());
 
@@ skipping 1628 matching lines @@
 {
     visitor->trace(m_client);
 }
 
 DEFINE_TRACE(HTMLMediaElement::AudioSourceProviderImpl)
 {
     visitor->trace(m_client);
 }
 #endif
 
+void HTMLMediaElement::autoplayExperimentInstallEventListenerIfNeeded()
+{
+    // If we don't require visibility, then we don't need the listener.
+    if (!(m_autoplayExperimentMode & ExperimentIfVisible))
+        return;
+
+    if (document().domWindow() && !m_autoplayExperimentTouchListener)  {
+        m_autoplayExperimentTouchListener = adoptRef(new AutoplayExperimentTouchListener(this));
+        // Listen for events that might show a user-initiated scroll.  We
+        // don't try to catch programmatic scrolls right now.
+        document().domWindow()->addEventListener("touchend", m_autoplayExperimentTouchListener, false);
+        document().domWindow()->addEventListener("touchcancel", m_autoplayExperimentTouchListener, false);
+    }
 }
+
+void HTMLMediaElement::autoplayExperimentClearEventListenerIfNeeded()
+{
+    if (m_autoplayExperimentTouchListener) {
+        LocalDOMWindow* domWindow = document().domWindow();
+        if (domWindow) {
+            domWindow->removeEventListener("touchend", m_autoplayExperimentTouchListener, false);
+            domWindow->removeEventListener("touchcancel", m_autoplayExperimentTouchListener, false);
+        }
+        // Either way, clear our ref.
+        m_autoplayExperimentTouchListener.clear();
+    }
+}
+
+void HTMLMediaElement::beginPeriodicVisibilityCheck()
+{
+    // Note that a visibility check might already be in progress.
+    // Always reset the span of the checks to maximum.
+    m_autoplayVisibilityTimerSpan = visibilityCheckDuration;
+
+    // If the timer isn't active, then fire the timer immediately to reset
+    // it.  We might just setOneShot(0).
+    if (!m_autoplayVisibilityTimer.isActive()) {
+        visibilityTimerFired(0);
+    }
+}
+
+void HTMLMediaElement::visibilityTimerFired(Timer<HTMLMediaElement>*)
+{
+    const LocalDOMWindow* domWindow = document().domWindow();
+    if (!domWindow)
+        return;
+
+    const int currentScrollX = domWindow->scrollX();
+    const int currentScrollY = domWindow->scrollY();
+
+    if (currentScrollX != m_autoplayLastScrollX
+        || currentScrollY != m_autoplayLastScrollY) {
+        // Still scrolling, so wait a bit more.
+        m_autoplayLastScrollX = currentScrollX;
+        m_autoplayLastScrollY = currentScrollY;
+
+        // Reset the timer to check again if we haven't tried for long enough.
+        m_autoplayVisibilityTimerSpan -= visibilityTimerPollDelay;
+        if (m_autoplayVisibilityTimerSpan >= 0) {
+            m_autoplayVisibilityTimer.startOneShot(visibilityTimerPollDelay, FROM_HERE);
+        }
+    } else {
+        // No longer scrolling, so check visibility and stop.
+        autoplayExperimentMaybeStartPlaying();
+    }
+}
+
+bool HTMLMediaElement::autoplayExperimentIsVisible()
+{
+    // We could check for eligibility here, but we skip it.  Some of our
+    // callers need to do it separately, and we don't want to check more
+    // than we need to.
+
+    // If visibility isn't required, then it's visible enough.
+    if (!(m_autoplayExperimentMode & ExperimentIfVisible))
+        return true;
+
+    // Autoplay is requested, and we're willing to override if the visibility
+    // requirements are met.
+
+    // Check visibility.

[philipj_slow, 2015/08/05 10:03:11]

A page visibility check is missing, this ought to pass even if the tab is in the
background. Make sure to have a separate flag for page visibity, maybe
"pagevisible" and "inviewport" or something?

[liberato (no reviews please), 2015/08/06 06:37:57]

i don't think it'll ever play unless in the foreground, at least on android. 
i'll add a comment to that effect and rename 'visible' to make it more clear
that it's viewport.

[philipj_slow, 2015/08/13 09:27:53]

What is it that would prevent playback when not in the foreground? Video is
paused when going into the background, but I'm not sure if there's anything to
prevent from starting to play when the user gesture requirement is gone. For
audio, there's no pausing in the background either.

[liberato (no reviews please), 2015/09/01 06:54:19]

Done.

+    const LocalDOMWindow* domWindow = document().domWindow();
+    if (!domWindow)
+        return false;
+
+    FloatRect us(offsetLeft(), offsetTop(), clientWidth(), clientHeight());
+    FloatRect screen(domWindow->scrollX(), domWindow->scrollY(), domWindow->innerWidth(), domWindow->innerHeight());
+
+    return screen.contains(us);
+}
+
+bool HTMLMediaElement::autoplayExperimentMaybeStartPlaying()
+{
+    // Make sure that we're eligible and visible.
+    if (!autoplayExperimentIsEligible() || !autoplayExperimentIsVisible()) {
+        return false;
+    }
+
+    // Start playing!
+    autoplayExperimentPrepareToPlay(AutoplayExperimentStartedByScroll);
+    // Why are we always preparing?  Just go!

[philipj_slow, 2015/08/05 10:03:12]

Is this a question to the reviewer, or a rhetorical question? FWIW, I'm not sure
when we need to prepare and not, so if you know, please document it :)

[liberato (no reviews please), 2015/08/06 06:37:57]

random movie quotes make reviewing more fun!

but yeah, i'd remove it before landing either way :)

[philipj_slow, 2015/08/13 09:27:53]

Ah, Spaceballs :)

+    playInternal();
+
+    return true;
+}
+
+bool HTMLMediaElement::autoplayExperimentIsEligible() const
+{
+    // If no user gesture is required, then the experiment doesn't apply.
+    // This is what prevents us from starting playback more than once.
+    // Since this flag is never set to true once it's cleared, it will block
+    // the autoplay experiment forever.
+    if (!m_userGestureRequiredForPlay)
+        return false;
+
+    if (m_autoplayExperimentMode == ExperimentOff)
+        return false;
+
+    // If nobody has requested playback, either by the autoplay attribute or
+    // a play() call, then do nothing.
+    if (!m_autoplayExperimentPlayPending && !autoplay())
+        return false;
+
+    // If the video is already playing, then do nothing.  Note that there
+    // is not a path where a user gesture is required but the video is
+    // playing.  However, we check for completeness.
+    if (!m_paused)
+        return false;
+
+    // Note that the viewport test always returns false on desktop, which is
+    // why video-autoplay-experiment.html doesn't check -ifmobile .
+    if (m_autoplayExperimentMode & ExperimentIfMobile) {
+        if (!document().viewportDescription().isLegacyViewportType())
+            return false;
+    }
+
+    if (m_autoplayExperimentMode & ExperimentIfMuted) {
+        // If media is muted, then autoplay when it comes into view.
+        return fastHasAttribute(mutedAttr) || m_muted;
+    }
+
+    // Autoplay when it comes into view (if needed), maybe muted.
+    return true;
+}
+
+void HTMLMediaElement::autoplayExperimentMuteIfNeeded()
+{
+    if (m_autoplayExperimentMode & ExperimentPlayMuted) {
+        m_muted = true;
+        updateVolume();
+    }
+}
+
+void HTMLMediaElement::autoplayExperimentPrepareToPlay(AutoplayMetrics metric)
+{
+    recordAutoplayMetric(metric);
+
+    // This also causes !autoplayExperimentIsEligible, so that we don't
+    // allow autoplay more than once.
+    m_userGestureRequiredForPlay = false;
+
+    m_autoplayExperimentStartedByExperiment = true;
+    autoplayExperimentClearEventListenerIfNeeded();
+    autoplayExperimentMuteIfNeeded();
+
+    // Record that this autoplayed without a user gesture.  This is normally
+    // set when we discover an autoplay attribute, but we include all cases
+    // where playback started without a user gesture, e.g., play().
+    m_initialPlayWithoutUserGestures = true;
+}
+
+}