Add srcObject attribute of type MediaStream to HTMLMediaElement.

third_party/WebKit/Source/core/html/HTMLMediaElement.cpp

 /*
  * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 65 matching lines @@
 #include "platform/RuntimeEnabledFeatures.h"
 #include "platform/UserGestureIndicator.h"
 #include "platform/audio/AudioBus.h"
 #include "platform/audio/AudioSourceProviderClient.h"
 #include "platform/graphics/GraphicsLayer.h"
 #include "platform/weborigin/SecurityOrigin.h"
 #include "public/platform/Platform.h"
 #include "public/platform/WebAudioSourceProvider.h"
 #include "public/platform/WebContentDecryptionModule.h"
 #include "public/platform/WebInbandTextTrack.h"
+#include "public/platform/WebMediaStream.h"
 #include "public/platform/modules/remoteplayback/WebRemotePlaybackClient.h"
 #include "public/platform/modules/remoteplayback/WebRemotePlaybackState.h"
 #include "wtf/CurrentTime.h"
 #include "wtf/MathExtras.h"
 #include "wtf/text/CString.h"
 #include <limits>
 
 #ifndef LOG_MEDIA_EVENTS
 // Default to not logging events because so many are generated they can overwhelm the rest of
 // the logging.
@@ skipping 520 matching lines @@
     return new LayoutMedia(this);
 }
 
 Node::InsertionNotificationRequest HTMLMediaElement::insertedInto(ContainerNode* insertionPoint)
 {
     WTF_LOG(Media, "HTMLMediaElement::insertedInto(%p, %p)", this, insertionPoint);
 
     HTMLElement::insertedInto(insertionPoint);
     if (insertionPoint->inDocument()) {
         UseCounter::count(document(), UseCounter::HTMLMediaElementInDocument);
-        if (!getAttribute(srcAttr).isEmpty() && m_networkState == NETWORK_EMPTY) {
+        if (m_networkState == NETWORK_EMPTY && (!getAttribute(srcAttr).isEmpty() || m_source.isMediaProviderObject())) {

[philipj_slow, 2016/04/06 11:58:42]

Hmm, why this change? This block of code isn't per spec, did something break if
you didn't extend it to srcObject?

[Guido Urdaneta, 2016/04/06 16:34:54]

Nothing broke with or without this change.
wolenetz@ suggested this might be necessary and it made sense to me.
I reverted it for now.

[philipj_slow, 2016/04/07 15:38:11]

What could now be broken when using srcObject is moving a media element between
documents. Even with the change it wouldn't work per spec, but it would work a
little bit. So if you want to write a test for it and think it makes sense, I'm
OK with that.

[Guido Urdaneta, 2016/04/08 10:58:47]

I think it makes sense to add the srcObject check to the condition. However, I
was unable to hit this condition. Whenever I tried moving an element with
nonempty src or srcObject, the networkState was never NETWORK_EMPTY.
Nevertheless, I added a layout test for moving an element with an srcObject to a
new document. It works regardless of this change.

             m_ignorePreloadNone = false;
             invokeLoadAlgorithm();
         }
     }
 
     return InsertionShouldCallDidNotifySubtreeInsertions;
 }
 
 void HTMLMediaElement::didNotifySubtreeInsertionsToDocument()
 {
@@ skipping 74 matching lines @@
 MediaError* HTMLMediaElement::error() const
 {
     return m_error;
 }
 
 void HTMLMediaElement::setSrc(const AtomicString& url)
 {
     setAttribute(srcAttr, url);
 }
 
+void HTMLMediaElement::setSrcObject(const WebMediaElementSource& srcObject)
+{
+    // The srcObject IDL attribute, on setting, must set the element's assigned
+    // media provider object to the new value, and then invoke the element's
+    // media element load algorithm.
+    m_source = srcObject;

[philipj_slow, 2016/04/06 11:58:42]

Looking at this I'm thinking that it'd be cleaner if this is called just
m_srcObject and that before setting it you
ASSERT(srcObject.isMediaProviderObject()).

[Guido Urdaneta, 2016/04/06 16:34:55]

Done.

+    invokeLoadAlgorithm();
+}
+
 HTMLMediaElement::NetworkState HTMLMediaElement::getNetworkState() const
 {
     return m_networkState;
 }
 
 String HTMLMediaElement::canPlayType(const String& mimeType) const
 {
     WebMimeRegistry::SupportsType support = supportsType(ContentType(mimeType));
     String canPlay;
 
@@ skipping 158 matching lines @@
         }
     }
 
     selectMediaResource();
 }
 
 void HTMLMediaElement::selectMediaResource()
 {
     WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p)", this);
 
-    enum Mode { attribute, children };
+    enum Mode { Object, Attribute, Children, Nothing };
+    Mode mode = Nothing;
 
-    // 3 - If the media element has a src attribute, then let mode be attribute.
-    Mode mode = attribute;
-    if (!fastHasAttribute(srcAttr)) {
-        // Otherwise, if the media element does not have a src attribute but has a source
-        // element child, then let mode be children and let candidate be the first such
-        // source element child in tree order.
-        if (HTMLSourceElement* element = Traversal<HTMLSourceElement>::firstChild(*this)) {
-            mode = children;
-            m_nextChildNodeToConsider = element;
-            m_currentSourceNode = nullptr;
-        } else {
-            // Otherwise the media element has neither a src attribute nor a source element
-            // child: set the networkState to NETWORK_EMPTY, and abort these steps; the
-            // synchronous section ends.
-            m_loadState = WaitingForSource;
-            setShouldDelayLoadEvent(false);
-            setNetworkState(NETWORK_EMPTY);
-            updateDisplayState();
+    // 6 - If the media element has an assigned media provider object, then let
+    //     mode be object.
+    if (m_source.isMediaProviderObject()) {
+        mode = Object;
+    } else if (fastHasAttribute(srcAttr)) {
+        // Otherwise, if the media element has no assigned media provider object
+        // but has a src attribute, then let mode be attribute.
+        mode = Attribute;
+    } else if (HTMLSourceElement* element = Traversal<HTMLSourceElement>::firstChild(*this)) {
+        // Otherwise, if the media element does not have an assigned media
+        // provider object and does not have a src attribute, but does have a
+        // source element child, then let mode be children and let candidate be
+        // the first such source element child in tree order.
+        mode = Children;
+        m_nextChildNodeToConsider = element;
+        m_currentSourceNode = nullptr;
+    } else {
+        // Otherwise the media element has no assigned media provider object and
+        // has neither a src attribute nor a source element child: set the
+        // networkState to NETWORK_EMPTY, and abort these steps; the synchronous
+        // section ends.
+        m_loadState = WaitingForSource;
+        setShouldDelayLoadEvent(false);
+        setNetworkState(NETWORK_EMPTY);
+        updateDisplayState();
 
-            WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), nothing to load", this);
-            return;
-        }
-    }
-
-    // 4 - Set the media element's delaying-the-load-event flag to true (this delays the load event),
-    // and set its networkState to NETWORK_LOADING.
-    setShouldDelayLoadEvent(true);
-    setNetworkState(NETWORK_LOADING);
-
-    // 5 - Queue a task to fire a simple event named loadstart at the media element.
-    scheduleEvent(EventTypeNames::loadstart);
-
-    // 6 - If mode is attribute, then run these substeps
-    if (mode == attribute) {
-        m_loadState = LoadingFromSrcAttr;
-
-        const AtomicString& srcValue = fastGetAttribute(srcAttr);
-        // If the src attribute's value is the empty string ... jump down to the failed step below
-        if (srcValue.isEmpty()) {
-            mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
-            WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), empty 'src'", this);
-            return;
-        }
-
-        KURL mediaURL = document().completeURL(srcValue);
-        if (!isSafeToLoadURL(mediaURL, Complain)) {
-            mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
-            return;
-        }
-
-        // No type is available when the url comes from the 'src' attribute so MediaPlayer
-        // will have to pick a media engine based on the file extension.
-        ContentType contentType((String()));
-        loadResource(mediaURL, contentType);
-        WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), using 'src' attribute url", this);
+        WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), nothing to load", this);
         return;
     }
 
-    // Otherwise, the source elements will be used
-    loadNextSourceChild();
+    // 7 - Set the media element's networkState to NETWORK_LOADING.
+    setNetworkState(NETWORK_LOADING);
+
+    // 8 - Queue a task to fire a simple event named loadstart at the media element.
+    scheduleEvent(EventTypeNames::loadstart);
+
+    // 9 - Run the appropriate steps...
+    switch (mode) {
+    case Object:
+        loadSourceFromObject();
+        WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), using 'srcObject' attribute", this);
+        break;
+    case Attribute:
+        loadSourceFromAttribute();
+        WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), using 'src' attribute url", this);
+        break;
+    case Children:
+        loadNextSourceChild();
+        WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), using source element", this);
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+}
+
+void HTMLMediaElement::loadSourceFromObject()
+{
+    ASSERT(m_source.isMediaProviderObject());
+    m_loadState = LoadingFromSrcObject;
+
+    // No type is available when the resource comes from the 'srcObject'
+    // attribute.
+    ContentType contentType((String()));
+    loadResource(contentType);
+}
+
+void HTMLMediaElement::loadSourceFromAttribute()
+{
+    m_loadState = LoadingFromSrcAttr;
+    const AtomicString& srcValue = fastGetAttribute(srcAttr);
+
+    // If the src attribute's value is the empty string ... jump down to the failed step below
+    if (srcValue.isEmpty()) {
+        mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
+        WTF_LOG(Media, "HTMLMediaElement::selectMediaResource(%p), empty 'src'", this);
+        return;
+    }
+
+    KURL mediaURL = document().completeURL(srcValue);
+    if (!isSafeToLoadURL(mediaURL, Complain)) {
+        mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
+        return;
+    }
+
+    // No type is available when the url comes from the 'src' attribute so
+    // MediaPlayer will have to pick a media engine based on the file extension.
+    ContentType contentType((String()));
+    m_source = WebMediaElementSource(WebURL(mediaURL));

[philipj_slow, 2016/04/06 11:58:42]

Here, rather than setting m_source/m_srcObject, can you add back loadResource's
first argument and just pass it in here? I guess it can be const
WebMediaPlayerSource& and the WebMediaPlayer doesn't get any ownership, so it
has to extract the wrapped source from it before returning.

This should force the code into being more like the spec, it won't be possible
to keep some extra state in m_source/m_srcObject, which could be tempting even
if you haven't done it here.

[Guido Urdaneta, 2016/04/06 16:34:55]

Done.

+    loadResource(contentType);
 }
 
 void HTMLMediaElement::loadNextSourceChild()
 {
     ContentType contentType((String()));
     KURL mediaURL = selectNextSourceChild(&contentType, Complain);
     if (!mediaURL.isValid()) {
         waitForSourceChange();
         return;
     }
 
     // Reset the MediaPlayer and MediaSource if any
     resetMediaPlayerAndMediaSource();
 
     m_loadState = LoadingFromSourceElement;
-    loadResource(mediaURL, contentType);
+    m_source = WebMediaElementSource(WebURL(mediaURL));

[philipj_slow, 2016/04/06 11:58:42]

Ditto.

[Guido Urdaneta, 2016/04/06 16:34:54]

Done.

+    loadResource(contentType);
 }
 
-void HTMLMediaElement::loadResource(const KURL& url, ContentType& contentType)
+void HTMLMediaElement::loadResource(ContentType& contentType)
 {
     ASSERT(isMainThread());
-    ASSERT(isSafeToLoadURL(url, Complain));
-
-    WTF_LOG(Media, "HTMLMediaElement::loadResource(%p, %s, %s)", this, urlForLoggingMedia(url).utf8().data(), contentType.raw().utf8().data());
+    KURL url;
+    if (m_source.isURL()) {
+        url = m_source.getAsURL();
+        ASSERT(isSafeToLoadURL(url, Complain));
+        WTF_LOG(Media, "HTMLMediaElement::loadResource(%p, %s, %s)", this, urlForLoggingMedia(url).utf8().data(), contentType.raw().utf8().data());
+    }
 
     LocalFrame* frame = document().frame();
     if (!frame) {
         mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
         return;
     }
 
     // The resource fetch algorithm
     setNetworkState(NETWORK_LOADING);
 
@@ skipping 16 matching lines @@
     setPlayerPreload();
 
     if (fastHasAttribute(mutedAttr))
         m_muted = true;
     updateVolume();
 
     ASSERT(!m_mediaSource);
 
     bool attemptLoad = true;
 
-    if (url.protocolIs(mediaSourceBlobProtocol)) {
-        if (isMediaStreamURL(url.getString())) {
+    bool isBlobProtocol = m_source.isMediaProviderObject() || url.protocolIs(mediaSourceBlobProtocol);

[philipj_slow, 2016/04/06 11:58:42]

I guess isObjectOrBlobURL or something would be more accurate?

[Guido Urdaneta, 2016/04/06 16:34:55]

Done.

+    if (isBlobProtocol) {
+        bool isMediaStream = m_source.isMediaStream() || (m_source.isURL() && isMediaStreamURL(url.getString()));
+        if (isMediaStream) {
             m_autoplayHelper->removeUserGestureRequirement(GesturelessPlaybackEnabledByStream);
         } else {
             m_mediaSource = HTMLMediaSource::lookup(url.getString());
 
             if (m_mediaSource) {
                 if (!m_mediaSource->attachToElement(this)) {
                     // Forget our reference to the MediaSource, so we leave it alone
                     // while processing remainder of load failure.
                     m_mediaSource = nullptr;
                     attemptLoad = false;
                 }
             }
         }
     }
 
-    if (attemptLoad && canLoadURL(url, contentType)) {
+    bool canLoadResource = m_source.isMediaProviderObject() || canLoadURL(url, contentType);
+    if (attemptLoad && canLoadResource) {
         ASSERT(!webMediaPlayer());
 
         if (effectivePreloadType() == WebMediaPlayer::PreloadNone) {
             WTF_LOG(Media, "HTMLMediaElement::loadResource(%p) : Delaying load because preload == 'none'", this);
             deferLoad();
         } else {
             startPlayerLoad();
         }
     } else {
         mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
     }
 
     // If there is no poster to display, allow the media engine to render video frames as soon as
     // they are available.
     updateDisplayState();
 
     if (layoutObject())
         layoutObject()->updateFromElement();
 }
 
 void HTMLMediaElement::startPlayerLoad()
 {
     ASSERT(!m_webMediaPlayer);
-    // Filter out user:pass as those two URL components aren't
-    // considered for media resource fetches (including for the CORS
-    // use-credentials mode.) That behavior aligns with Gecko, with IE
-    // being more restrictive and not allowing fetches to such URLs.
-    //
-    // Spec reference: http://whatwg.org/c/#concept-media-load-resource
-    //
-    // FIXME: when the HTML spec switches to specifying resource
-    // fetches in terms of Fetch (http://fetch.spec.whatwg.org), and
-    // along with that potentially also specifying a setting for its
-    // 'authentication flag' to control how user:pass embedded in a
-    // media resource URL should be treated, then update the handling
-    // here to match.
-    KURL requestURL = m_currentSrc;
-    if (!requestURL.user().isEmpty())
-        requestURL.setUser(String());
-    if (!requestURL.pass().isEmpty())
-        requestURL.setPass(String());
+
+    if (m_source.isURL()) {

[philipj_slow, 2016/04/06 11:58:42]

This will have to change with my proposed changes to m_srcObject. Will you be
able to use m_mode?

[Guido Urdaneta, 2016/04/06 16:34:55]

Done.

+        // Filter out user:pass as those two URL components aren't
+        // considered for media resource fetches (including for the CORS
+        // use-credentials mode.) That behavior aligns with Gecko, with IE
+        // being more restrictive and not allowing fetches to such URLs.
+        //
+        // Spec reference: http://whatwg.org/c/#concept-media-load-resource
+        //
+        // FIXME: when the HTML spec switches to specifying resource
+        // fetches in terms of Fetch (http://fetch.spec.whatwg.org), and
+        // along with that potentially also specifying a setting for its
+        // 'authentication flag' to control how user:pass embedded in a
+        // media resource URL should be treated, then update the handling
+        // here to match.
+        KURL requestURL = m_currentSrc;
+        if (!requestURL.user().isEmpty())
+            requestURL.setUser(String());
+        if (!requestURL.pass().isEmpty())
+            requestURL.setPass(String());
+
+        KURL kurl(ParsedURLString, requestURL);
+        m_source = WebMediaElementSource(WebURL(kurl));
+    }
 
     LocalFrame* frame = document().frame();
     // TODO(srirama.m): Figure out how frame can be null when
     // coming from executeDeferredLoad()
     if (!frame) {
         mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
         return;
     }
 
-    KURL kurl(ParsedURLString, requestURL);
-    m_webMediaPlayer = frame->loader().client()->createWebMediaPlayer(*this, kurl, this);
+    m_webMediaPlayer = frame->loader().client()->createWebMediaPlayer(*this, m_source, this);
     if (!m_webMediaPlayer) {
         mediaLoadingFailed(WebMediaPlayer::NetworkStateFormatError);
         return;
     }
 
     if (layoutObject())
         layoutObject()->setShouldDoFullPaintInvalidation();
     // Make sure if we create/re-create the WebMediaPlayer that we update our wrapper.
     m_audioSourceProvider.wrap(m_webMediaPlayer->getAudioSourceProvider());
     m_webMediaPlayer->setVolume(effectiveMediaVolume());
 
     m_webMediaPlayer->setPoster(posterImageURL());
 
     m_webMediaPlayer->setPreload(effectivePreloadType());
 
-    m_webMediaPlayer->load(loadType(), kurl, corsMode());
+    m_webMediaPlayer->load(loadType(), m_source, corsMode());
 
     if (isFullscreen()) {
         // This handles any transition to or from fullscreen overlay mode.
         frame->chromeClient().enterFullScreenForElement(this);
     }
 }
 
 void HTMLMediaElement::setPlayerPreload()
 {
     if (m_webMediaPlayer)
@@ skipping 72 matching lines @@
     }
     ASSERT(m_deferredLoadState == WaitingForStopDelayingLoadEventTask);
     m_deferredLoadState = WaitingForTrigger;
 }
 
 WebMediaPlayer::LoadType HTMLMediaElement::loadType() const
 {
     if (m_mediaSource)
         return WebMediaPlayer::LoadTypeMediaSource;
 
+    if (m_source.isMediaStream())
+        return WebMediaPlayer::LoadTypeMediaStream;
+
     if (isMediaStreamURL(m_currentSrc.getString()))
         return WebMediaPlayer::LoadTypeMediaStream;
 
     return WebMediaPlayer::LoadTypeURL;
 }
 
 bool HTMLMediaElement::textTracksAreReady() const
 {
     // 4.8.10.12.1 Text track model
     // ...
@@ skipping 2663 matching lines @@
 }
 
 #if !ENABLE(OILPAN)
 WeakPtr<HTMLMediaElement> HTMLMediaElement::createWeakPtr()
 {
     return m_weakPtrFactory.createWeakPtr();
 }
 #endif
 
 } // namespace blink