Implement AudioTrack, AudioTrackList, VideoTrack, and VideoTrackList

Source/core/html/HTMLMediaElement.cpp

Index: Source/core/html/HTMLMediaElement.cpp
diff --git a/Source/core/html/HTMLMediaElement.cpp b/Source/core/html/HTMLMediaElement.cpp
index 8fd4af03e209fc46bb74cb770e05de7302d4e013..5299ac91deb05f9826edf61cf9426873c9d50514 100644
--- a/Source/core/html/HTMLMediaElement.cpp
+++ b/Source/core/html/HTMLMediaElement.cpp
@@ -51,9 +51,13 @@
 #include "core/html/MediaFragmentURIParser.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/shadow/MediaControls.h"
+#include "core/html/track/AudioTrack.h"
+#include "core/html/track/AudioTrackList.h"
 #include "core/html/track/InbandTextTrack.h"
 #include "core/html/track/TextTrackCueList.h"
 #include "core/html/track/TextTrackList.h"
+#include "core/html/track/VideoTrack.h"
+#include "core/html/track/VideoTrackList.h"
 #include "core/loader/FrameLoader.h"
 #include "core/rendering/RenderVideo.h"
 #include "core/rendering/RenderView.h"
@@ -85,6 +89,7 @@ using namespace std;
 using blink::WebInbandTextTrack;
 using blink::WebMediaPlayer;
 using blink::WebMimeRegistry;
+using blink::WebMediaPlayerClient;
 
 namespace WebCore {
 
@@ -169,6 +174,52 @@ private:
     HTMLMediaElement* m_mediaElement;
 };
 
+static const AtomicString& AudioKindToString(WebMediaPlayerClient::AudioTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::AudioTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::AudioTrackKindAlternative:
+        return AudioTrack::alternativeKeyword();
+    case WebMediaPlayerClient::AudioTrackKindDescriptions:
+        return AudioTrack::descriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMain:
+        return AudioTrack::mainKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMainDescriptions:
+        return AudioTrack::mainDescriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindTranslation:
+        return AudioTrack::translationKeyword();
+    case WebMediaPlayerClient::AudioTrackKindCommentary:
+        return AudioTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
+static const AtomicString& VideoKindToString(WebMediaPlayerClient::VideoTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::VideoTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::VideoTrackKindAlternative:
+        return VideoTrack::alternativeKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCaptions:
+        return VideoTrack::captionsKeyword();
+    case WebMediaPlayerClient::VideoTrackKindMain:
+        return VideoTrack::mainKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSign:
+        return VideoTrack::signKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSubtitles:
+        return VideoTrack::subtitlesKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCommentary:
+        return VideoTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
 static bool canLoadURL(const KURL& url, const ContentType& contentType, const String& keySystem)
 {
     DEFINE_STATIC_LOCAL(const String, codecs, ("codecs"));
@@ -240,6 +291,7 @@ HTMLMediaElement::HTMLMediaElement(const QualifiedName& tagName, Document& docum
     , m_loadTimer(this, &HTMLMediaElement::loadTimerFired)
     , m_progressEventTimer(this, &HTMLMediaElement::progressEventTimerFired)
     , m_playbackProgressTimer(this, &HTMLMediaElement::playbackProgressTimerFired)
+    , m_audioTracksTimer(this, &HTMLMediaElement::audioTracksTimerFired)
     , m_playedTimeRanges()
     , m_asyncEventQueue(GenericEventQueue::create(this))
     , m_playbackRate(1.0f)
@@ -283,6 +335,8 @@ HTMLMediaElement::HTMLMediaElement(const QualifiedName& tagName, Document& docum
     , m_haveVisibleTextTrack(false)
     , m_processingPreferenceChange(false)
     , m_lastTextTrackUpdateTime(-1)
+    , m_audioTracks(AudioTrackList::create(*this))
+    , m_videoTracks(VideoTrackList::create(*this))
     , m_textTracks(nullptr)
     , m_ignoreTrackDisplayUpdate(0)
 #if ENABLE(WEB_AUDIO)
@@ -317,6 +371,9 @@ HTMLMediaElement::~HTMLMediaElement()
         m_textTracks->clearOwner();
 #endif
 
+    m_audioTracks->shutdown();
+    m_videoTracks->shutdown();
+
     if (m_mediaController) {
         m_mediaController->removeMediaElement(this);
         m_mediaController = nullptr;
@@ -619,6 +676,7 @@ void HTMLMediaElement::prepareForLoad()
     m_loadTimer.stop();
     // FIXME: Figure out appropriate place to reset LoadTextTrackResource if necessary and set m_pendingActionFlags to 0 here.
     m_pendingActionFlags &= ~LoadMediaResource;
+    m_audioTracksTimer.stop();
     m_sentEndEvent = false;
     m_sentStalledEvent = false;
     m_haveFiredLoadedData = false;
@@ -1602,7 +1660,12 @@ void HTMLMediaElement::setReadyState(MediaPlayer::ReadyState state)
     }
 
     if (m_readyState >= HAVE_METADATA && oldState < HAVE_METADATA) {
+        createPlaceholderTracksIfNecessary();
+
         prepareMediaFragmentURI();
+
+        selectInitialTracksIfNecessary();
+
         scheduleEvent(EventTypeNames::durationchange);
         if (isHTMLVideoElement(*this))
             scheduleEvent(EventTypeNames::resize);
@@ -2219,6 +2282,115 @@ void HTMLMediaElement::togglePlayState()
     }
 }
 
+AudioTrackList& HTMLMediaElement::audioTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_audioTracks;
+}
+
+void HTMLMediaElement::audioTrackChanged()
+{
+    WTF_LOG(Media, "HTMLMediaElement::audioTrackChanged()");
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    audioTracks().scheduleChangeEvent();
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.audioTracks attribute is added.
+
+    if (!m_audioTracksTimer.isActive())
+        m_audioTracksTimer.startOneShot(0, FROM_HERE);
+}
+
+void HTMLMediaElement::audioTracksTimerFired(Timer<HTMLMediaElement>*)
+{
+    if (!webMediaPlayer())
+        return;
+
+    Vector<blink::WebTrackId> enabledTrackIds;
+    for (unsigned i = 0; i < audioTracks().length(); ++i) {
+        RefPtr<AudioTrack> track = audioTracks().anonymousIndexedGetter(i);
+        if (track->enabled())
+            enabledTrackIds.append(track->webId());
+    }
+
+    webMediaPlayer()->enabledAudioTracksChanged(enabledTrackIds);
+}
+
+blink::WebTrackId HTMLMediaElement::addAudioTrack(const String& id, blink::WebMediaPlayerClient::AudioTrackKind kind, const AtomicString& label, const AtomicString& language, bool enabled)
+{
+    AtomicString kindString = AudioKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addAudioTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), enabled);
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return 0;
+
+    RefPtr<AudioTrack> audioTrack = AudioTrack::create(id, kindString, label, language);
+    audioTracks().add(audioTrack.get());
+
+    if (enabled)
+        audioTrack->setEnabled(true);
+
+    return audioTrack->webId();
+}
+
+void HTMLMediaElement::removeAudioTrack(blink::WebTrackId webId)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeAudioTrack()");
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    audioTracks().remove(webId);
+}
+
+VideoTrackList& HTMLMediaElement::videoTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_videoTracks;
+}
+
+void HTMLMediaElement::selectedVideoTrackChanged(blink::WebTrackId* selectedTrackId)
+{
+    WTF_LOG(Media, "HTMLMediaElement::selectedVideoTrackChanged()");
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    videoTracks().trackSelected(selectedTrackId);
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.videoTracks attribute is added.
+
+    if (webMediaPlayer())
+        webMediaPlayer()->selectedVideoTrackChanged(selectedTrackId);
+}
+
+blink::WebTrackId HTMLMediaElement::addVideoTrack(const String& id, blink::WebMediaPlayerClient::VideoTrackKind kind, const AtomicString& label, const AtomicString& language, bool selected)
+{
+    AtomicString kindString = VideoKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addVideoTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), selected);
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return 0;
+
+    RefPtr<VideoTrack> videoTrack = VideoTrack::create(id, kindString, label, language);
+    videoTracks().add(videoTrack.get());
+
+    if (selected)
+        videoTrack->setSelected(true);

[landell, 2014/05/17 06:15:28]

I see one potential problem here. Seems as this call can generate a
selectedVideoTrackChanged call on the media player before the player has
received the WebTrackId for the track. This can be worked around in the media
player but makes the interface a bit unintuitive IMO.

[philipj_slow, 2014/05/20 09:05:13]

Good catch, that's a pretty terrible API.

I can see two sane fixes:

1. When the backend sources a new video track which is selected, just make it
the selected track without a callback to say "yes I did what you asked."

2. Use a single timeout for all tracks, which would make the callback happen at
a later point. The callback wouldn't be very useful, it would just be a side
effect of triggering all track changes together.

Preferences?

+
+    return videoTrack->webId();
+}
+
+void HTMLMediaElement::removeVideoTrack(blink::WebTrackId webId)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeVideoTrack()");
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    videoTracks().remove(webId);

[acolwell GONE FROM CHROMIUM, 2014/05/16 19:47:56]

Still need to check for removing the selected track here.

+}
+
 void HTMLMediaElement::mediaPlayerDidAddTextTrack(WebInbandTextTrack* webTrack)
 {
     // 4.8.10.12.2 Sourcing in-band text tracks
@@ -2288,11 +2460,17 @@ void HTMLMediaElement::removeTextTrack(TextTrack* track)
 
 void HTMLMediaElement::forgetResourceSpecificTracks()
 {
+    // Implements the "forget the media element's media-resource-specific tracks" algorithm.
+    // The order is explicitly specified as text, then audio, and finally video. Also
+    // 'removetrack' events should not be fired.
     if (m_textTracks) {
         TrackDisplayUpdateScope scope(this);
         m_textTracks->removeAllInbandTracks();
         closeCaptionTracksChanged();
     }
+
+    m_audioTracks->removeAll();
+    m_videoTracks->removeAll();
 }
 
 PassRefPtrWillBeRawPtr<TextTrack> HTMLMediaElement::addTextTrack(const AtomicString& kind, const AtomicString& label, const AtomicString& language, ExceptionState& exceptionState)
@@ -3118,6 +3296,7 @@ void HTMLMediaElement::clearMediaPlayer(int flags)
 
     stopPeriodicTimers();
     m_loadTimer.stop();
+    m_audioTracksTimer.stop();
 
     m_pendingActionFlags &= ~flags;
     m_loadState = WaitingForSource;
@@ -3560,6 +3739,8 @@ void HTMLMediaElement::prepareMediaFragmentURI()
     } else
         m_fragmentEndTime = MediaPlayer::invalidTime();
 
+    // FIXME: Add support for selecting tracks by ID with the Media Fragments track dimension.
+
     if (m_fragmentStartTime != MediaPlayer::invalidTime() && m_readyState < HAVE_FUTURE_DATA)
         prepareToPlay();
 }
@@ -3620,4 +3801,32 @@ void HTMLMediaElement::trace(Visitor* visitor)
     HTMLElement::trace(visitor);
 }
 
+void HTMLMediaElement::createPlaceholderTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Create a placeholder audio track if |m_player| says it has audio but it didn't explicitly announce the tracks.
+    if (m_player->hasAudio() && !audioTracks().length())
+        addAudioTrack("audio", WebMediaPlayerClient::AudioTrackKindMain, "Audio Track", "", true);
+
+    // Create a placeholder video track if |m_player| says it has video but it didn't explicitly announce the tracks.
+    if (m_player->hasVideo() && !videoTracks().length())
+        addVideoTrack("video", WebMediaPlayerClient::VideoTrackKindMain, "Video Track", "", true);
+}
+
+void HTMLMediaElement::selectInitialTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Enable the first audio track if an audio track hasn't been enabled yet.
+    if (audioTracks().length() > 0 && !audioTracks().hasEnabledTrack())
+        audioTracks().anonymousIndexedGetter(0)->setEnabled(true);
+
+    // Select the first video track if a video track hasn't been selected yet.
+    if (videoTracks().length() > 0 && videoTracks().selectedIndex() == -1)
+        videoTracks().anonymousIndexedGetter(0)->setSelected(true);
+}
+
 }