Initial implementation of AudioTrack, AudioTrackList, VideoTrack, and VideoTrackList.

Source/core/html/HTMLMediaElement.cpp

 /*
  * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 33 matching lines @@
 #include "core/frame/UseCounter.h"
 #include "core/frame/csp/ContentSecurityPolicy.h"
 #include "core/html/HTMLMediaSource.h"
 #include "core/html/HTMLSourceElement.h"
 #include "core/html/HTMLTrackElement.h"
 #include "core/html/MediaController.h"
 #include "core/html/MediaError.h"
 #include "core/html/MediaFragmentURIParser.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/shadow/MediaControls.h"
+#include "core/html/track/AudioTrack.h"
+#include "core/html/track/AudioTrackList.h"
 #include "core/html/track/InbandTextTrack.h"
 #include "core/html/track/TextTrackCueList.h"
 #include "core/html/track/TextTrackList.h"
+#include "core/html/track/VideoTrack.h"
+#include "core/html/track/VideoTrackList.h"
 #include "core/loader/FrameLoader.h"
 #include "core/rendering/RenderVideo.h"
 #include "core/rendering/RenderView.h"
 #include "core/rendering/compositing/RenderLayerCompositor.h"
 #include "platform/ContentType.h"
 #include "platform/Language.h"
 #include "platform/Logging.h"
 #include "platform/MIMETypeFromURL.h"
 #include "platform/MIMETypeRegistry.h"
 #include "platform/NotImplemented.h"
@@ skipping 11 matching lines @@
 
 #if ENABLE(WEB_AUDIO)
 #include "platform/audio/AudioSourceProvider.h"
 #include "modules/webaudio/MediaElementAudioSourceNode.h"
 #endif
 
 using namespace std;
 using blink::WebInbandTextTrack;
 using blink::WebMediaPlayer;
 using blink::WebMimeRegistry;
+using blink::WebMediaPlayerClient;
 
 namespace WebCore {
 
 #if !LOG_DISABLED
 static String urlForLoggingMedia(const KURL& url)
 {
     static const unsigned maximumURLLengthForLogging = 128;
 
     if (url.string().length() < maximumURLLengthForLogging)
         return url.string();
@@ skipping 58 matching lines @@
     ~TrackDisplayUpdateScope()
     {
         ASSERT(m_mediaElement);
         m_mediaElement->endIgnoringTrackDisplayUpdateRequests();
     }
 
 private:
     HTMLMediaElement* m_mediaElement;
 };
 
+static const AtomicString& AudioKindToString(WebMediaPlayerClient::AudioTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::AudioTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::AudioTrackKindAlternative:
+        return AudioTrack::alternativeKeyword();
+    case WebMediaPlayerClient::AudioTrackKindDescriptions:
+        return AudioTrack::descriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMain:
+        return AudioTrack::mainKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMainDescriptions:
+        return AudioTrack::mainDescriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindTranslation:
+        return AudioTrack::translationKeyword();
+    case WebMediaPlayerClient::AudioTrackKindCommentary:
+        return AudioTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
+static const AtomicString& VideoKindToString(WebMediaPlayerClient::VideoTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::VideoTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::VideoTrackKindAlternative:
+        return VideoTrack::alternativeKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCaptions:
+        return VideoTrack::captionsKeyword();
+    case WebMediaPlayerClient::VideoTrackKindMain:
+        return VideoTrack::mainKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSign:
+        return VideoTrack::signKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSubtitles:
+        return VideoTrack::subtitlesKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCommentary:
+        return VideoTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
 static bool canLoadURL(const KURL& url, const ContentType& contentType, const String& keySystem)
 {
     DEFINE_STATIC_LOCAL(const String, codecs, ("codecs"));
 
     String contentMIMEType = contentType.type().lower();
     String contentTypeCodecs = contentType.parameter(codecs);
 
     // If the MIME type is missing or is not meaningful, try to figure it out from the URL.
     if (contentMIMEType.isEmpty() || contentMIMEType == "application/octet-stream" || contentMIMEType == "text/plain") {
         if (url.protocolIsData())
@@ skipping 51 matching lines @@
 {
     return s_mediaStreamRegistry ? s_mediaStreamRegistry->contains(url) : false;
 }
 
 HTMLMediaElement::HTMLMediaElement(const QualifiedName& tagName, Document& document)
     : HTMLElement(tagName, document)
     , ActiveDOMObject(&document)
     , m_loadTimer(this, &HTMLMediaElement::loadTimerFired)
     , m_progressEventTimer(this, &HTMLMediaElement::progressEventTimerFired)
     , m_playbackProgressTimer(this, &HTMLMediaElement::playbackProgressTimerFired)
+    , m_audioTracksTimer(this, &HTMLMediaElement::audioTracksTimerFired)
     , m_playedTimeRanges()
     , m_asyncEventQueue(GenericEventQueue::create(this))
     , m_playbackRate(1.0f)
     , m_defaultPlaybackRate(1.0f)
     , m_networkState(NETWORK_EMPTY)
     , m_readyState(HAVE_NOTHING)
     , m_readyStateMaximum(HAVE_NOTHING)
     , m_volume(1.0f)
     , m_lastSeekTime(0)
     , m_previousProgressTime(numeric_limits<double>::max())
@@ skipping 24 matching lines @@
     , m_sentEndEvent(false)
     , m_pausedInternal(false)
     , m_closedCaptionsVisible(false)
     , m_completelyLoaded(false)
     , m_havePreparedToPlay(false)
     , m_delayingLoadForPreloadNone(false)
     , m_tracksAreReady(true)
     , m_haveVisibleTextTrack(false)
     , m_processingPreferenceChange(false)
     , m_lastTextTrackUpdateTime(-1)
+    , m_audioTracks(AudioTrackList::create(*this))
+    , m_videoTracks(VideoTrackList::create(*this))
     , m_textTracks(nullptr)
     , m_ignoreTrackDisplayUpdate(0)
 #if ENABLE(WEB_AUDIO)
     , m_audioSourceNode(0)
 #endif
 {
     ASSERT(RuntimeEnabledFeatures::mediaEnabled());
 
     WTF_LOG(Media, "HTMLMediaElement::HTMLMediaElement");
     ScriptWrappable::init(this);
@@ skipping 12 matching lines @@
 {
     WTF_LOG(Media, "HTMLMediaElement::~HTMLMediaElement");
 
     m_asyncEventQueue->close();
 
     setShouldDelayLoadEvent(false);
 
     if (m_textTracks)
         m_textTracks->clearOwner();
 
+    m_audioTracks->shutdown();
+    m_videoTracks->shutdown();
+
     if (m_mediaController) {
         m_mediaController->removeMediaElement(this);
         m_mediaController = nullptr;
     }
 
     closeMediaSource();
 
     removeElementFromDocumentMap(this, &document());
 
     // Destroying the player may cause a resource load to be canceled,
@@ skipping 39 matching lines @@
     // previous document. A proper fix would provide a mechanism to allow this
     // object to refresh the MediaPlayer's LocalFrame and FrameLoader references on
     // document changes so that playback can be resumed properly.
     userCancelledLoad();
 
     // Decrement the load event delay count on oldDocument now that m_player has been destroyed
     // and there is no risk of dispatching a load event from within the destructor.
     oldDocument.decrementLoadEventDelayCount();
 
     ActiveDOMObject::didMoveToNewExecutionContext(&document());
+
     HTMLElement::didMoveToNewDocument(oldDocument);
 }
 
 bool HTMLMediaElement::hasCustomFocusLogic() const
 {
     return true;
 }
 
 bool HTMLMediaElement::supportsFocus() const
 {
@@ skipping 213 matching lines @@
 
 void HTMLMediaElement::prepareForLoad()
 {
     WTF_LOG(Media, "HTMLMediaElement::prepareForLoad");
 
     // Perform the cleanup required for the resource load algorithm to run.
     stopPeriodicTimers();
     m_loadTimer.stop();
     // FIXME: Figure out appropriate place to reset LoadTextTrackResource if necessary and set m_pendingActionFlags to 0 here.
     m_pendingActionFlags &= ~LoadMediaResource;
+    m_audioTracksTimer.stop();
     m_sentEndEvent = false;
     m_sentStalledEvent = false;
     m_haveFiredLoadedData = false;
     m_completelyLoaded = false;
     m_havePreparedToPlay = false;
     m_displayMode = Unknown;
 
     // 1 - Abort any already-running instance of the resource selection algorithm for this element.
     m_loadState = WaitingForSource;
     m_currentSourceNode = nullptr;
@@ skipping 966 matching lines @@
             finishSeek();
     } else {
         if (wasPotentiallyPlaying && m_readyState < HAVE_FUTURE_DATA) {
             // 4.8.10.8
             scheduleTimeupdateEvent(false);
             scheduleEvent(EventTypeNames::waiting);
         }
     }
 
     if (m_readyState >= HAVE_METADATA && oldState < HAVE_METADATA) {
+        createPlaceholderTracksIfNecessary();
+
         prepareMediaFragmentURI();
+
+        selectInitialTracksIfNecessary();
+
         scheduleEvent(EventTypeNames::durationchange);
         if (isHTMLVideoElement(*this))
             scheduleEvent(EventTypeNames::resize);
         scheduleEvent(EventTypeNames::loadedmetadata);
         if (hasMediaControls())
             mediaControls()->reset();
         if (renderer())
             renderer()->updateFromElement();
     }
 
@@ skipping 577 matching lines @@
         m_lastTimeUpdateEventWallTime = now;
         m_lastTimeUpdateEventMovieTime = movieTime;
     }
 }
 
 bool HTMLMediaElement::canPlay() const
 {
     return paused() || ended() || m_readyState < HAVE_METADATA;
 }
 
+
 void HTMLMediaElement::togglePlayState()
 {
     ASSERT(controls());
     // The activation behavior of a media element that is exposing a user interface to the user
     if (m_mediaController) {
         if (m_mediaController->isRestrained())
             m_mediaController->play();
         else if (m_mediaController->paused())
             m_mediaController->unpause();
         else
             m_mediaController->pause();
     } else {
         if (paused())
             play();
         else
             pause();
     }
+
+AudioTrackList& HTMLMediaElement::audioTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_audioTracks;
+}
+
+void HTMLMediaElement::audioTrackChanged(const String& audioTrackID)
+{
+    WTF_LOG(Media, "HTMLMediaElement::audioTrackChanged('%s')", audioTrackID.ascii().data());
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    ASSERT(!audioTrackID.isEmpty());
+
+    audioTracks().scheduleChangeEvent();
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.audioTracks attribute is added.
+
+    if (!m_audioTracksTimer.isActive())
+        m_audioTracksTimer.startOneShot(0, FROM_HERE);
+}
+
+void HTMLMediaElement::addAudioTrack(const String& id, blink::WebMediaPlayerClient::AudioTrackKind kind, const AtomicString& label, const AtomicString& language, bool enabled)
+{
+    AtomicString kindString = AudioKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addAudioTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), enabled);
+    ASSERT(!id.isEmpty());

[philipj_slow, 2014/04/23 12:08:07]

We've made a little discovery on this point. The spec says "The AudioTrack.id
and VideoTrack.id attributes must return the identifier of the track, if it has
one, or the empty string otherwise." In other words, we can't use the id to
identify tracks (like in removeAudioTrack below) and the assert is invalid.

It's not exactly clear if just using the index as an identifier everywhere is
going to work, but if that doesn't work an internal unique handle for each track
could be added.

+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    RefPtr<AudioTrack> audioTrack = AudioTrack::create(this, id, kindString, label, language);
+    audioTracks().add(audioTrack.get());
+
+    if (enabled)
+        audioTrack->setEnabled(true);
+}
+
+void HTMLMediaElement::removeAudioTrack(const String& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeAudioTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    audioTracks().remove(id);
+}
+
+VideoTrackList& HTMLMediaElement::videoTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_videoTracks;
+}
+
+void HTMLMediaElement::selectedVideoTrackChanged(const String& selectedTrackID)
+{
+    WTF_LOG(Media, "HTMLMediaElement::selectedVideoTrackChanged('%s')", selectedTrackID.ascii().data());
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    String unselectedTrackID;
+
+    int oldSelectedIndex = videoTracks().selectedIndex();
+    if (oldSelectedIndex != -1)
+        unselectedTrackID = videoTracks().anonymousIndexedGetter(oldSelectedIndex)->id();
+
+    ASSERT(unselectedTrackID != selectedTrackID);
+    ASSERT(!unselectedTrackID.isEmpty() || !selectedTrackID.isEmpty());
+
+    videoTracks().trackSelected(selectedTrackID);
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.videoTracks attribute is added.
+
+    if (webMediaPlayer())
+        webMediaPlayer()->selectedVideoTrackChanged(selectedTrackID);
+}
+
+void HTMLMediaElement::addVideoTrack(const String& id, blink::WebMediaPlayerClient::VideoTrackKind kind, const AtomicString& label, const AtomicString& language, bool selected)
+{
+    AtomicString kindString = VideoKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addVideoTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), selected);
+    ASSERT(!id.isEmpty());
+
+    RefPtr<VideoTrack> videoTrack = VideoTrack::create(this, id, kindString, label, language);
+    videoTracks().add(videoTrack.get());
+
+    if (selected)
+        videoTrack->setSelected(true);
+}
+
+void HTMLMediaElement::removeVideoTrack(const String& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeVideoTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    videoTracks().remove(id);
 }
 
 void HTMLMediaElement::mediaPlayerDidAddTextTrack(WebInbandTextTrack* webTrack)
 {
     if (!RuntimeEnabledFeatures::videoTrackEnabled())
         return;
 
     // 4.8.10.12.2 Sourcing in-band text tracks
     // 1. Associate the relevant data with a new text track and its corresponding new TextTrack object.
     RefPtr<InbandTextTrack> textTrack = InbandTextTrack::create(document(), webTrack);
@@ skipping 57 matching lines @@
 void HTMLMediaElement::removeTextTrack(TextTrack* track)
 {
     TrackDisplayUpdateScope scope(this);
     m_textTracks->remove(track);
 
     closeCaptionTracksChanged();
 }
 
 void HTMLMediaElement::forgetResourceSpecificTracks()
 {
+    // Implements the "forget the media element's media-resource-specific tracks" algorithm.
+    // The order is explicitly specified as text, then audio, and finally video. Also
+    // 'removetrack' events should not be fired.
     if (m_textTracks) {
         TrackDisplayUpdateScope scope(this);
         m_textTracks->removeAllInbandTracks();
         closeCaptionTracksChanged();
     }
+
+    m_audioTracks->removeAll();
+    m_videoTracks->removeAll();
 }
 
 PassRefPtr<TextTrack> HTMLMediaElement::addTextTrack(const AtomicString& kind, const AtomicString& label, const AtomicString& language, ExceptionState& exceptionState)
 {
     ASSERT(RuntimeEnabledFeatures::videoTrackEnabled());
 
     // 4.8.10.12.4 Text track API
     // The addTextTrack(kind, label, language) method of media elements, when invoked, must run the following steps:
 
     // 1. If kind is not one of the following strings, then throw a SyntaxError exception and abort these steps
@@ skipping 831 matching lines @@
     forgetResourceSpecificTracks();
 
     closeMediaSource();
 
     m_delayingLoadForPreloadNone = false;
 
     clearMediaPlayerAndAudioSourceProviderClient();
 
     stopPeriodicTimers();
     m_loadTimer.stop();
+    m_audioTracksTimer.stop();
 
     m_pendingActionFlags &= ~flags;
     m_loadState = WaitingForSource;
 
     if (m_textTracks)
         configureTextTrackDisplay(AssumeNoVisibleChange);
 }
 
 void HTMLMediaElement::stop()
 {
@@ skipping 437 matching lines @@
         m_fragmentStartTime = MediaPlayer::invalidTime();
 
     double end = fragmentParser.endTime();
     if (end != MediaFragmentURIParser::invalidTimeValue() && end > 0 && end > m_fragmentStartTime) {
         m_fragmentEndTime = end;
         if (m_fragmentEndTime > dur)
             m_fragmentEndTime = dur;
     } else
         m_fragmentEndTime = MediaPlayer::invalidTime();
 
+    // FIXME: Add support for selecting tracks by ID with the Media Fragments track dimension.
+
     if (m_fragmentStartTime != MediaPlayer::invalidTime() && m_readyState < HAVE_FUTURE_DATA)
         prepareToPlay();
 }
 
 void HTMLMediaElement::applyMediaFragmentURI()
 {
     if (m_fragmentStartTime != MediaPlayer::invalidTime()) {
         m_sentEndEvent = false;
         UseCounter::count(document(), UseCounter::HTMLMediaElementSeekToFragmentStart);
         seek(m_fragmentStartTime, IGNORE_EXCEPTION);
@@ skipping 38 matching lines @@
 void HTMLMediaElement::mediaPlayerMediaSourceOpened(blink::WebMediaSource* webMediaSource)
 {
     m_mediaSource->setWebMediaSourceAndOpen(adoptPtr(webMediaSource));
 }
 
 bool HTMLMediaElement::isInteractiveContent() const
 {
     return fastHasAttribute(controlsAttr);
 }
 
+void HTMLMediaElement::createPlaceholderTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Create a placeholder audio track if |m_player| says it has audio but it didn't explicitly announce the tracks.
+    if (m_player->hasAudio() && !audioTracks().length())
+        addAudioTrack("audio", WebMediaPlayerClient::AudioTrackKindMain, "Audio Track", "", true);
+
+    // Create a placeholder video track if |m_player| says it has video but it didn't explicitly announce the tracks.
+    if (m_player->hasVideo() && !videoTracks().length())
+        addVideoTrack("video", WebMediaPlayerClient::VideoTrackKindMain, "Video Track", "", true);
 }
+
+void HTMLMediaElement::selectInitialTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Enable the first audio track if an audio track hasn't been enabled yet.
+    if (audioTracks().length() > 0 && !audioTracks().hasEnabledTrack())
+        audioTracks().anonymousIndexedGetter(0)->setEnabled(true);
+
+    // Select the first video track if a video track hasn't been selected yet.
+    if (videoTracks().length() > 0 && videoTracks().selectedIndex() == -1)
+        videoTracks().anonymousIndexedGetter(0)->setSelected(true);
+}
+
+void HTMLMediaElement::audioTracksTimerFired(Timer<HTMLMediaElement>*)
+{
+    if (!webMediaPlayer())
+        return;
+
+    Vector<String> enabledTrackIDs;
+    for (unsigned i = 0; i < audioTracks().length(); ++i) {
+        RefPtr<AudioTrack> track = audioTracks().anonymousIndexedGetter(i);
+        if (track->enabled())
+            enabledTrackIDs.append(track->id());
+    }
+
+    webMediaPlayer()->enabledAudioTracksChanged(enabledTrackIDs);
+}
+
+}