Initial implementation of AudioTrack, AudioTrackList, VideoTrack, and VideoTrackList.

Source/core/html/HTMLMediaElement.cpp

 /*
  * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 34 matching lines @@
 #include "core/frame/UseCounter.h"
 #include "core/frame/csp/ContentSecurityPolicy.h"
 #include "core/html/HTMLMediaSource.h"
 #include "core/html/HTMLSourceElement.h"
 #include "core/html/HTMLTrackElement.h"
 #include "core/html/MediaController.h"
 #include "core/html/MediaError.h"
 #include "core/html/MediaFragmentURIParser.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/shadow/MediaControls.h"
+#include "core/html/track/AudioTrack.h"
+#include "core/html/track/AudioTrackList.h"
 #include "core/html/track/InbandTextTrack.h"
 #include "core/html/track/TextTrackCueList.h"
 #include "core/html/track/TextTrackList.h"
+#include "core/html/track/VideoTrack.h"
+#include "core/html/track/VideoTrackList.h"
 #include "core/loader/FrameLoader.h"
 #include "core/rendering/RenderVideo.h"
 #include "core/rendering/RenderView.h"
 #include "core/rendering/compositing/RenderLayerCompositor.h"
 #include "platform/ContentType.h"
 #include "platform/Language.h"
 #include "platform/Logging.h"
 #include "platform/MIMETypeFromURL.h"
 #include "platform/MIMETypeRegistry.h"
 #include "platform/NotImplemented.h"
@@ skipping 10 matching lines @@
 #include "wtf/text/CString.h"
 
 #if ENABLE(WEB_AUDIO)
 #include "platform/audio/AudioSourceProvider.h"
 #include "modules/webaudio/MediaElementAudioSourceNode.h"
 #endif
 
 using namespace std;
 using blink::WebInbandTextTrack;
 using blink::WebMimeRegistry;
+using blink::WebMediaPlayerClient;
 
 namespace WebCore {
 
 #if !LOG_DISABLED
 static String urlForLoggingMedia(const KURL& url)
 {
     static const unsigned maximumURLLengthForLogging = 128;
 
     if (url.string().length() < maximumURLLengthForLogging)
         return url.string();
@@ skipping 58 matching lines @@
     ~TrackDisplayUpdateScope()
     {
         ASSERT(m_mediaElement);
         m_mediaElement->endIgnoringTrackDisplayUpdateRequests();
     }
 
 private:
     HTMLMediaElement* m_mediaElement;
 };
 
+static const AtomicString& VideoKindToString(WebMediaPlayerClient::VideoTrackKind kind)

[philipj_slow, 2014/03/09 09:04:14]

Switch the order of this and the AudioKindToString for consistency?

[acolwell GONE FROM CHROMIUM, 2014/03/18 22:02:15]

Done.

+{
+    switch (kind) {
+    case WebMediaPlayerClient::VideoTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::VideoTrackKindAlternative:
+        return VideoTrack::alternativeKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCaptions:
+        return VideoTrack::captionsKeyword();
+    case WebMediaPlayerClient::VideoTrackKindMain:
+        return VideoTrack::mainKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSign:
+        return VideoTrack::signKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSubtitles:
+        return VideoTrack::subtitlesKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCommentary:
+        return VideoTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
+static const AtomicString& AudioKindToString(WebMediaPlayerClient::AudioTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::AudioTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::AudioTrackKindAlternative:
+        return AudioTrack::alternativeKeyword();
+    case WebMediaPlayerClient::AudioTrackKindDescriptions:
+        return AudioTrack::descriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMain:
+        return AudioTrack::mainKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMainDescriptions:
+        return AudioTrack::mainDescriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindTranslation:
+        return AudioTrack::translationKeyword();
+    case WebMediaPlayerClient::AudioTrackKindCommentary:
+        return AudioTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
 static bool canLoadURL(const KURL& url, const ContentType& contentType, const String& keySystem)
 {
     DEFINE_STATIC_LOCAL(const String, codecs, ("codecs"));
 
     String contentMIMEType = contentType.type().lower();
     String contentTypeCodecs = contentType.parameter(codecs);
 
     // If the MIME type is missing or is not meaningful, try to figure it out from the URL.
     if (contentMIMEType.isEmpty() || contentMIMEType == "application/octet-stream" || contentMIMEType == "text/plain") {
         if (url.protocolIsData())
@@ skipping 126 matching lines @@
 {
     WTF_LOG(Media, "HTMLMediaElement::~HTMLMediaElement");
 
     m_asyncEventQueue->close();
 
     setShouldDelayLoadEvent(false);
 
     if (m_textTracks)
         m_textTracks->clearOwner();
 
+    if (m_audioTracks)
+        m_audioTracks->shutdown();

[philipj_slow, 2014/03/09 09:04:14]

This will cancel pending events, and I think it's going to be observable: Load a
resource with a single video track and then disable that track. This schedules a
change event. Add an event handler, remove the src attribute, call load(),
remove all references to the media element and trigger a GC. This code will run
and the event will never fire.

The root cause is that "The task source for the tasks listed in this section is
the DOM manipulation task source." but we're using an m_asyncEventQueue per
track list. In additional to missing events, it makes it pretty easy to create a
test case where the event order is wrong.

Again, since you're reusing existing code it's up to you how much to fix in this
review. I can file bugs for what remains when the review is done.

[acolwell GONE FROM CHROMIUM, 2014/03/18 22:02:15]

I've removed the m_asyncEventQueue->close() from shutdown() so the events won't
be cancelled. That was a mistake on my part.
I missed the part about all this being from the DOM manipulation task source so
I figured it would be ok to have different queues. After reading up on this I
see that using different queues is the wrong thing to do.
In general, I'd like to avoid introducing known bugs with new code. Filing bugs
for the existing code sounds good to me. 

[philipj_slow, 2014/03/20 16:17:39]

Bug for TextTrackList:
https://code.google.com/p/chromium/issues/detail?id=354414

+
+    if (m_videoTracks)
+        m_videoTracks->shutdown();
+
     if (m_mediaController) {
         m_mediaController->removeMediaElement(this);
         m_mediaController = nullptr;
     }
 
     closeMediaSource();
 
     removeElementFromDocumentMap(this, &document());
 
     // Destroying the player may cause a resource load to be canceled,
@@ skipping 39 matching lines @@
     // previous document. A proper fix would provide a mechanism to allow this
     // object to refresh the MediaPlayer's LocalFrame and FrameLoader references on
     // document changes so that playback can be resumed properly.
     userCancelledLoad();
 
     // Decrement the load event delay count on oldDocument now that m_player has been destroyed
     // and there is no risk of dispatching a load event from within the destructor.
     oldDocument.decrementLoadEventDelayCount();
 
     ActiveDOMObject::didMoveToNewExecutionContext(&document());
+
+    // Notify track lists so they don't stop firing events when moved to a new document

[philipj_slow, 2014/03/09 09:04:14]

Can you add a test case for this? Seems like something other implementors might
get wrong as well.

+    // and the original document gets destroyed.
+    if (m_audioTracks)
+        m_audioTracks->didMoveToNewExecutionContext(&document());
+
+    if (m_videoTracks)
+        m_videoTracks->didMoveToNewExecutionContext(&document());
+
     HTMLElement::didMoveToNewDocument(oldDocument);
 }
 
 bool HTMLMediaElement::hasCustomFocusLogic() const
 {
     return true;
 }
 
 bool HTMLMediaElement::supportsFocus() const
 {
@@ skipping 1192 matching lines @@
             finishSeek();
     } else {
         if (wasPotentiallyPlaying && m_readyState < HAVE_FUTURE_DATA) {
             // 4.8.10.8
             scheduleTimeupdateEvent(false);
             scheduleEvent(EventTypeNames::waiting);
         }
     }
 
     if (m_readyState >= HAVE_METADATA && oldState < HAVE_METADATA) {
+        createPlaceholderTracksIfNecessary();
+
         prepareMediaFragmentURI();
+
+        selectInitialTracksIfNecessary();
+
         scheduleEvent(EventTypeNames::durationchange);
         if (isVideo())
             scheduleEvent(EventTypeNames::resize);
         scheduleEvent(EventTypeNames::loadedmetadata);
         if (hasMediaControls())
             mediaControls()->reset();
         if (renderer())
             renderer()->updateFromElement();
     }
 
@@ skipping 600 matching lines @@
         m_lastTimeUpdateEventWallTime = now;
         m_lastTimeUpdateEventMovieTime = movieTime;
     }
 }
 
 bool HTMLMediaElement::canPlay() const
 {
     return paused() || ended() || m_readyState < HAVE_METADATA;
 }
 
+AudioTrackList& HTMLMediaElement::audioTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    if (!m_audioTracks)
+        m_audioTracks = AudioTrackList::create(this);
+
+    return *m_audioTracks;
+}
+
+void HTMLMediaElement::audioTrackChanged(const AtomicString& audioTrackID, bool enabled)
+{
+    WTF_LOG(Media, "HTMLMediaElement::audioTrackChanged('%s', %d)", audioTrackID.ascii().data(), enabled);
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    ASSERT(!audioTrackID.isEmpty());
+
+    audioTracks().scheduleChangeEvent();
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.audioTracks attribute is added.
+
+    if (webMediaPlayer())
+        webMediaPlayer()->enabledAudioTrackChange(audioTrackID, enabled);

[philipj_slow, 2014/03/09 09:04:14]

This API seems to make gapless track switching hard. I think that awaiting a
stable state and then notifying the backend about which tracks should be enabled
and disabled would make it easier. A timeout in the backend wouldn't work,
because the script can disable a track and then take time (without returning to
the event loop) to decide which new track to enable, and the backend has no idea
how long to wait.

+}
+
+VideoTrackList& HTMLMediaElement::videoTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    if (!m_videoTracks)
+        m_videoTracks = VideoTrackList::create(this);
+
+    return *m_videoTracks;
+}
+
+void HTMLMediaElement::selectedVideoTrackChanged(const AtomicString& selectedTrackID)
+{
+    WTF_LOG(Media, "HTMLMediaElement::selectedVideoTrackChanged('%s')", selectedTrackID.ascii().data());
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    AtomicString unselectedTrackID;
+
+    int oldSelectedIndex = videoTracks().selectedIndex();
+    if (oldSelectedIndex != -1)
+        unselectedTrackID = videoTracks().anonymousIndexedGetter(oldSelectedIndex)->id();
+
+    ASSERT(unselectedTrackID != selectedTrackID);
+
+    videoTracks().trackSelected(selectedTrackID);
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.videoTracks attribute is added.
+
+    ASSERT(!unselectedTrackID.isEmpty() || !selectedTrackID.isEmpty());
+
+    if (webMediaPlayer())
+        webMediaPlayer()->selectedVideoTrackChange(unselectedTrackID, selectedTrackID);
+}
+
+void HTMLMediaElement::addVideoTrack(const AtomicString& id, blink::WebMediaPlayerClient::VideoTrackKind kind, const AtomicString& label, const AtomicString& language, bool selected)
+{
+    AtomicString kindString = VideoKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addVideoTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), selected);
+    ASSERT(!id.isEmpty());

[philipj_slow, 2014/03/09 09:04:14]

Also assert that the id is unique? selectedVideoTrackChanged already ASSERTS it
in an indirect manner.

+
+    RefPtr<VideoTrack> videoTrack = VideoTrack::create(this, id, kindString, label, language);
+    videoTracks().add(videoTrack.get());
+
+    if (selected)
+        videoTrack->setSelected(true);

[philipj_slow, 2014/03/09 09:04:14]

This will fire a change event for any new track... "Whenever a track in a
VideoTrackList that was previously not selected is selected, the user agent must
queue a task to fire a simple event named change at the VideoTrackList object."
Since the VideoTrack previously didn't exist, "previously not selected" becomes
non-sensical, I've filed a spec bug for clarification:
https://www.w3.org/Bugs/Public/show_bug.cgi?id=24977

+}
+
+void HTMLMediaElement::removeVideoTrack(const AtomicString& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeVideoTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    videoTracks().remove(id);
+}
+
+void HTMLMediaElement::addAudioTrack(const AtomicString& id, blink::WebMediaPlayerClient::AudioTrackKind kind, const AtomicString& label, const AtomicString& language, bool enabled)

[philipj_slow, 2014/03/09 09:04:14]

Move these up together with the other *AudioTrack functions?

[acolwell GONE FROM CHROMIUM, 2014/03/18 22:02:15]

Done.

+{
+    AtomicString kindString = AudioKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addAudioTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), enabled);
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    RefPtr<AudioTrack> audioTrack = AudioTrack::create(this, id, kindString, label, language);
+    audioTracks().add(audioTrack.get());
+
+    if (enabled)
+        audioTrack->setEnabled(true);
+}
+
+void HTMLMediaElement::removeAudioTrack(const AtomicString& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeAudioTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    audioTracks().remove(id);
+}
+
 void HTMLMediaElement::mediaPlayerDidAddTextTrack(WebInbandTextTrack* webTrack)
 {
     if (!RuntimeEnabledFeatures::videoTrackEnabled())
         return;
 
     // 4.8.10.12.2 Sourcing in-band text tracks
     // 1. Associate the relevant data with a new text track and its corresponding new TextTrack object.
     RefPtr<InbandTextTrack> textTrack = InbandTextTrack::create(document(), webTrack);
 
     // 2. Set the new text track's kind, label, and language based on the semantics of the relevant data,
@@ skipping 55 matching lines @@
 void HTMLMediaElement::removeTextTrack(TextTrack* track)
 {
     TrackDisplayUpdateScope scope(this);
     m_textTracks->remove(track);
 
     closeCaptionTracksChanged();
 }
 
 void HTMLMediaElement::forgetResourceSpecificTracks()
 {
+    // Implements the "forget the media element's media-resource-specific tracks" algorithm.
+    // The order is explicitly specified as text, then audio, and finally video. Also

[philipj_slow, 2014/03/09 09:04:14]

Hmm, are there any events that fire that make it possible to tell the
difference? If there are, test please :)

[acolwell GONE FROM CHROMIUM, 2014/03/18 22:02:15]

I don't believe so. I just put the comment here to keep this code matching the
spec, just in case something comes up later.

[philipj_slow, 2014/03/20 16:17:39]

Great!

+    // 'removetrack' events should not be fired.
     if (m_textTracks) {
         TrackDisplayUpdateScope scope(this);
         m_textTracks->removeAllInbandTracks();
         closeCaptionTracksChanged();
     }
+
+    if (m_audioTracks)
+        m_audioTracks->removeAll();
+
+    if (m_videoTracks)
+        m_videoTracks->removeAll();
 }
 
 PassRefPtr<TextTrack> HTMLMediaElement::addTextTrack(const AtomicString& kind, const AtomicString& label, const AtomicString& language, ExceptionState& exceptionState)
 {
     ASSERT(RuntimeEnabledFeatures::videoTrackEnabled());
 
     // 4.8.10.12.4 Text track API
     // The addTextTrack(kind, label, language) method of media elements, when invoked, must run the following steps:
 
     // 1. If kind is not one of the following strings, then throw a SyntaxError exception and abort these steps
@@ skipping 1289 matching lines @@
         m_fragmentStartTime = MediaPlayer::invalidTime();
 
     double end = fragmentParser.endTime();
     if (end != MediaFragmentURIParser::invalidTimeValue() && end > 0 && end > m_fragmentStartTime) {
         m_fragmentEndTime = end;
         if (m_fragmentEndTime > dur)
             m_fragmentEndTime = dur;
     } else
         m_fragmentEndTime = MediaPlayer::invalidTime();
 
+    // FIXME: Add support for selecting tracks by ID with the Media Fragments track dimension.
+
     if (m_fragmentStartTime != MediaPlayer::invalidTime() && m_readyState < HAVE_FUTURE_DATA)
         prepareToPlay();
 }
 
 void HTMLMediaElement::applyMediaFragmentURI()
 {
     if (m_fragmentStartTime != MediaPlayer::invalidTime()) {
         m_sentEndEvent = false;
         UseCounter::count(document(), UseCounter::HTMLMediaElementSeekToFragmentStart);
         seek(m_fragmentStartTime, IGNORE_EXCEPTION);
@@ skipping 38 matching lines @@
 void HTMLMediaElement::mediaPlayerMediaSourceOpened(blink::WebMediaSource* webMediaSource)
 {
     m_mediaSource->setWebMediaSourceAndOpen(adoptPtr(webMediaSource));
 }
 
 bool HTMLMediaElement::isInteractiveContent() const
 {
     return fastHasAttribute(controlsAttr);
 }
 
+void HTMLMediaElement::createPlaceholderTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Create a placeholder audio track if |m_player| says it has audio but it didn't explicitly announce the tracks.
+    if (m_player->hasAudio() && !audioTracks().length())
+        addAudioTrack("audio", WebMediaPlayerClient::AudioTrackKindMain, "Audio Track", "", true);
+
+    // Create a placeholder video track if |m_player| says it has video but it didn't explicitly announce the tracks.
+    if (m_player->hasVideo() && !videoTracks().length())
+        addVideoTrack("video", WebMediaPlayerClient::VideoTrackKindMain, "Video Track", "", true);
 }
+
+void HTMLMediaElement::selectInitialTracksIfNecessary()

[philipj_slow, 2014/03/09 09:04:14]

Why is this needed? We can assert that at least one of the added audio/video
tracks were enabled. That'll certainly hold true for
createPlaceholderTracksIfNecessary() and I think we should demand it from the
real backends as well.

[acolwell GONE FROM CHROMIUM, 2014/03/18 22:02:15]

This was initially intended to house the logic for initially selecting tracks
based on user preferences, like preferred language, but only currently provides
the fallback "if nothing was selected" behavior. I think it is better to have
default selection logic here instead of in the backend since Blink is likely to
be more aware of what the default selected tracks should be.

[philipj_slow, 2014/03/20 16:17:39]

Ah, I didn't really consider what the low-level media formats look like, is it
the case that the media resource won't itself can't always specify a default
track? If so I agree that having the logic here makes sense. If there are *no*
formats we (want to) support that can specify a default track, I think just
removing the selected/enabled arguments from the platform layer would make the
responsibilities clearer.

+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Enable the first audio track if an audio track hasn't been enabled yet.
+    if (audioTracks().length() > 0 && !audioTracks().hasEnabledTrack())
+        audioTracks().anonymousIndexedGetter(0)->setEnabled(true);
+
+    // Select the first video track if a video track hasn't been selected yet.
+    if (videoTracks().length() > 0 && videoTracks().selectedIndex() == -1)
+        videoTracks().anonymousIndexedGetter(0)->setSelected(true);
+}
+
+}