Initial implementation of AudioTrack, AudioTrackList, VideoTrack, and VideoTrackList.

Source/core/html/HTMLMediaElement.cpp

 /*
  * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 34 matching lines @@
 #include "core/frame/UseCounter.h"
 #include "core/frame/csp/ContentSecurityPolicy.h"
 #include "core/html/HTMLMediaSource.h"
 #include "core/html/HTMLSourceElement.h"
 #include "core/html/HTMLTrackElement.h"
 #include "core/html/MediaController.h"
 #include "core/html/MediaError.h"
 #include "core/html/MediaFragmentURIParser.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/shadow/MediaControls.h"
+#include "core/html/track/AudioTrack.h"
+#include "core/html/track/AudioTrackList.h"
 #include "core/html/track/InbandTextTrack.h"
 #include "core/html/track/TextTrackCueList.h"
 #include "core/html/track/TextTrackList.h"
+#include "core/html/track/VideoTrack.h"
+#include "core/html/track/VideoTrackList.h"
 #include "core/loader/FrameLoader.h"
 #include "core/rendering/RenderVideo.h"
 #include "core/rendering/RenderView.h"
 #include "core/rendering/compositing/RenderLayerCompositor.h"
 #include "platform/ContentType.h"
 #include "platform/Language.h"
 #include "platform/Logging.h"
 #include "platform/MIMETypeFromURL.h"
 #include "platform/MIMETypeRegistry.h"
 #include "platform/NotImplemented.h"
@@ skipping 10 matching lines @@
 #include "wtf/text/CString.h"
 
 #if ENABLE(WEB_AUDIO)
 #include "platform/audio/AudioSourceProvider.h"
 #include "modules/webaudio/MediaElementAudioSourceNode.h"
 #endif
 
 using namespace std;
 using blink::WebInbandTextTrack;
 using blink::WebMimeRegistry;
+using blink::WebMediaPlayerClient;
 
 namespace WebCore {
 
 #if !LOG_DISABLED
 static String urlForLoggingMedia(const KURL& url)
 {
     static const unsigned maximumURLLengthForLogging = 128;
 
     if (url.string().length() < maximumURLLengthForLogging)
         return url.string();
@@ skipping 58 matching lines @@
     ~TrackDisplayUpdateScope()
     {
         ASSERT(m_mediaElement);
         m_mediaElement->endIgnoringTrackDisplayUpdateRequests();
     }
 
 private:
     HTMLMediaElement* m_mediaElement;
 };
 
+static AtomicString VideoKindToString(WebMediaPlayerClient::VideoTrackKind kind)

[Inactive, 2014/03/08 03:05:59]

Looks like this could return a const AtomicString&?

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Done.

+{
+    switch (kind) {
+    case WebMediaPlayerClient::VideoTrackKindNone:
+        return AtomicString();

[Inactive, 2014/03/08 03:05:59]

nullAtom? (same below)

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Done.

+        break;

[Inactive, 2014/03/08 03:05:59]

Why the breaks after the return statements? (Sorry if this is per coding style
and I am just not aware of it).

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Removed. I think an earlier version of this function assigned to a local
variable instead of returning.

+    case WebMediaPlayerClient::VideoTrackKindAlternative:
+        return VideoTrack::alternativeKeyword();
+        break;
+    case WebMediaPlayerClient::VideoTrackKindCaptions:
+        return VideoTrack::captionsKeyword();
+        break;
+    case WebMediaPlayerClient::VideoTrackKindMain:
+        return VideoTrack::mainKeyword();
+        break;
+    case WebMediaPlayerClient::VideoTrackKindSign:
+        return VideoTrack::signKeyword();
+        break;
+    case WebMediaPlayerClient::VideoTrackKindSubtitles:
+        return VideoTrack::subtitlesKeyword();
+        break;
+    case WebMediaPlayerClient::VideoTrackKindCommentary:
+        return VideoTrack::commentaryKeyword();
+        break;
+    }
+
+    ASSERT_NOT_REACHED();
+    return AtomicString();
+}
+
+static AtomicString AudioKindToString(WebMediaPlayerClient::AudioTrackKind kind)

[Inactive, 2014/03/08 03:05:59]

Looks like this could return a const AtomicString&?

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Done.

+{
+    switch (kind) {
+    case WebMediaPlayerClient::AudioTrackKindNone:
+        return AtomicString();
+        break;
+    case WebMediaPlayerClient::AudioTrackKindAlternative:
+        return AudioTrack::alternativeKeyword();
+        break;
+    case WebMediaPlayerClient::AudioTrackKindDescriptions:
+        return AudioTrack::descriptionsKeyword();
+        break;
+    case WebMediaPlayerClient::AudioTrackKindMain:
+        return AudioTrack::mainKeyword();
+        break;
+    case WebMediaPlayerClient::AudioTrackKindMainDescriptions:
+        return AudioTrack::mainDescriptionsKeyword();
+        break;
+    case WebMediaPlayerClient::AudioTrackKindTranslation:
+        return AudioTrack::translationKeyword();
+        break;
+    case WebMediaPlayerClient::AudioTrackKindCommentary:
+        return AudioTrack::commentaryKeyword();
+        break;
+    }
+
+    ASSERT_NOT_REACHED();
+    return AtomicString();
+}
+
 static bool canLoadURL(const KURL& url, const ContentType& contentType, const String& keySystem)
 {
     DEFINE_STATIC_LOCAL(const String, codecs, ("codecs"));
 
     String contentMIMEType = contentType.type().lower();
     String contentTypeCodecs = contentType.parameter(codecs);
 
     // If the MIME type is missing or is not meaningful, try to figure it out from the URL.
     if (contentMIMEType.isEmpty() || contentMIMEType == "application/octet-stream" || contentMIMEType == "text/plain") {
         if (url.protocolIsData())
@@ skipping 94 matching lines @@
     , m_sentStalledEvent(false)
     , m_sentEndEvent(false)
     , m_pausedInternal(false)
     , m_closedCaptionsVisible(false)
     , m_completelyLoaded(false)
     , m_havePreparedToPlay(false)
     , m_tracksAreReady(true)
     , m_haveVisibleTextTrack(false)
     , m_processingPreferenceChange(false)
     , m_lastTextTrackUpdateTime(-1)
+    , m_audioTracks(nullptr)

[Inactive, 2014/03/08 03:05:59]

Those are RefPtrs, not raw pointers. No need to explicitly initialize them.

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Done.

+    , m_videoTracks(nullptr)
     , m_textTracks(nullptr)
     , m_ignoreTrackDisplayUpdate(0)
 #if ENABLE(WEB_AUDIO)
     , m_audioSourceNode(0)
 #endif
 {
     ASSERT(RuntimeEnabledFeatures::mediaEnabled());
 
     WTF_LOG(Media, "HTMLMediaElement::HTMLMediaElement");
     ScriptWrappable::init(this);
@@ skipping 12 matching lines @@
 {
     WTF_LOG(Media, "HTMLMediaElement::~HTMLMediaElement");
 
     m_asyncEventQueue->close();
 
     setShouldDelayLoadEvent(false);
 
     if (m_textTracks)
         m_textTracks->clearOwner();
 
+    if (m_audioTracks)
+        m_audioTracks->shutdown();
+
+    if (m_videoTracks)
+        m_videoTracks->shutdown();
+
     if (m_mediaController) {
         m_mediaController->removeMediaElement(this);
         m_mediaController = nullptr;
     }
 
     closeMediaSource();
 
     removeElementFromDocumentMap(this, &document());
 
     // Destroying the player may cause a resource load to be canceled,
@@ skipping 39 matching lines @@
     // previous document. A proper fix would provide a mechanism to allow this
     // object to refresh the MediaPlayer's LocalFrame and FrameLoader references on
     // document changes so that playback can be resumed properly.
     userCancelledLoad();
 
     // Decrement the load event delay count on oldDocument now that m_player has been destroyed
     // and there is no risk of dispatching a load event from within the destructor.
     oldDocument.decrementLoadEventDelayCount();
 
     ActiveDOMObject::didMoveToNewExecutionContext(&document());
+
+    // Notify track lists so they don't stop firing events when moved to a new document
+    // and the original document gets destroyed.
+    if (m_audioTracks)
+        m_audioTracks->didMoveToNewExecutionContext(&document());
+
+    if (m_videoTracks)
+        m_videoTracks->didMoveToNewExecutionContext(&document());
+
     HTMLElement::didMoveToNewDocument(oldDocument);
 }
 
 bool HTMLMediaElement::hasCustomFocusLogic() const
 {
     return true;
 }
 
 bool HTMLMediaElement::supportsFocus() const
 {
@@ skipping 1192 matching lines @@
             finishSeek();
     } else {
         if (wasPotentiallyPlaying && m_readyState < HAVE_FUTURE_DATA) {
             // 4.8.10.8
             scheduleTimeupdateEvent(false);
             scheduleEvent(EventTypeNames::waiting);
         }
     }
 
     if (m_readyState >= HAVE_METADATA && oldState < HAVE_METADATA) {
+        createPlaceholderTracksIfNecessary();
+
         prepareMediaFragmentURI();
+
+        selectInitialTracksIfNecessary();
+
         scheduleEvent(EventTypeNames::durationchange);
         if (isVideo())
             scheduleEvent(EventTypeNames::resize);
         scheduleEvent(EventTypeNames::loadedmetadata);
         if (hasMediaControls())
             mediaControls()->reset();
         if (renderer())
             renderer()->updateFromElement();
     }
 
@@ skipping 600 matching lines @@
         m_lastTimeUpdateEventWallTime = now;
         m_lastTimeUpdateEventMovieTime = movieTime;
     }
 }
 
 bool HTMLMediaElement::canPlay() const
 {
     return paused() || ended() || m_readyState < HAVE_METADATA;
 }
 
+AudioTrackList* HTMLMediaElement::audioTracks()

[Inactive, 2014/03/08 03:05:59]

Could return a reference as it never returns null.

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Done.

+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    if (!m_audioTracks)
+        m_audioTracks = AudioTrackList::create(this);
+
+    return m_audioTracks.get();
+}
+
+void HTMLMediaElement::audioTrackChanged(const AtomicString& audioTrackID, bool enabled)
+{
+    WTF_LOG(Media, "HTMLMediaElement::audioTrackChanged('%s', %d)", audioTrackID.ascii().data(), enabled);
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    ASSERT(!audioTrackID.isEmpty());
+
+    audioTracks()->scheduleChangeEvent();
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.audioTracks attribute is added.
+
+    if (webMediaPlayer())
+        webMediaPlayer()->enabledAudioTrackChange(audioTrackID, enabled);
+}
+
+VideoTrackList* HTMLMediaElement::videoTracks()

[Inactive, 2014/03/08 03:05:59]

Could return a reference as it never returns null.

[acolwell GONE FROM CHROMIUM, 2014/03/08 19:56:42]

Done.

+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    if (!m_videoTracks)
+        m_videoTracks = VideoTrackList::create(this);
+
+    return m_videoTracks.get();
+}
+
+void HTMLMediaElement::selectedVideoTrackChanged(const AtomicString& selectedTrackID)
+{
+    WTF_LOG(Media, "HTMLMediaElement::selectedVideoTrackChanged('%s')", selectedTrackID.ascii().data());
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    AtomicString unselectedTrackID;
+
+    int oldSelectedIndex = videoTracks()->selectedIndex();
+    if (oldSelectedIndex != -1)
+        unselectedTrackID = videoTracks()->anonymousIndexedGetter(oldSelectedIndex)->id();
+
+    ASSERT(unselectedTrackID != selectedTrackID);
+
+    videoTracks()->trackSelected(selectedTrackID);
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.videoTracks attribute is added.
+
+    ASSERT(!unselectedTrackID.isEmpty() || !selectedTrackID.isEmpty());
+
+    if (webMediaPlayer())
+        webMediaPlayer()->selectedVideoTrackChange(unselectedTrackID, selectedTrackID);
+}
+
+void HTMLMediaElement::addVideoTrack(const AtomicString& id, blink::WebMediaPlayerClient::VideoTrackKind kind, const AtomicString& label, const AtomicString& language, bool selected)
+{
+    AtomicString kindString = VideoKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addVideoTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), selected);
+    ASSERT(!id.isEmpty());
+
+    RefPtr<VideoTrack> videoTrack = VideoTrack::create(this, id, kindString, label, language);
+    videoTracks()->add(videoTrack.get());
+
+    if (selected)
+        videoTrack->setSelected(true);
+}
+
+void HTMLMediaElement::removeVideoTrack(const AtomicString& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeVideoTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    videoTracks()->remove(id);
+}
+
+void HTMLMediaElement::addAudioTrack(const AtomicString& id, blink::WebMediaPlayerClient::AudioTrackKind kind, const AtomicString& label, const AtomicString& language, bool enabled)
+{
+    AtomicString kindString = AudioKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addAudioTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), enabled);
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    RefPtr<AudioTrack> audioTrack = AudioTrack::create(this, id, kindString, label, language);
+    audioTracks()->add(audioTrack.get());
+
+    if (enabled)
+        audioTrack->setEnabled(true);
+}
+
+void HTMLMediaElement::removeAudioTrack(const AtomicString& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeAudioTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    audioTracks()->remove(id);
+}
+
 void HTMLMediaElement::mediaPlayerDidAddTextTrack(WebInbandTextTrack* webTrack)
 {
     if (!RuntimeEnabledFeatures::videoTrackEnabled())
         return;
 
     // 4.8.10.12.2 Sourcing in-band text tracks
     // 1. Associate the relevant data with a new text track and its corresponding new TextTrack object.
     RefPtr<InbandTextTrack> textTrack = InbandTextTrack::create(document(), webTrack);
 
     // 2. Set the new text track's kind, label, and language based on the semantics of the relevant data,
@@ skipping 55 matching lines @@
 void HTMLMediaElement::removeTextTrack(TextTrack* track)
 {
     TrackDisplayUpdateScope scope(this);
     m_textTracks->remove(track);
 
     closeCaptionTracksChanged();
 }
 
 void HTMLMediaElement::forgetResourceSpecificTracks()
 {
+    // Implements the "forget the media element's media-resource-specific tracks" algorithm.
+    // The order is explicitly specified as text, then audio, and finally video. Also
+    // 'removetrack' events should not be fired.
     if (m_textTracks) {
         TrackDisplayUpdateScope scope(this);
         m_textTracks->removeAllInbandTracks();
         closeCaptionTracksChanged();
     }
+
+    if (m_audioTracks)
+        m_audioTracks->removeAll();
+
+    if (m_videoTracks)
+        m_videoTracks->removeAll();
 }
 
 PassRefPtr<TextTrack> HTMLMediaElement::addTextTrack(const AtomicString& kind, const AtomicString& label, const AtomicString& language, ExceptionState& exceptionState)
 {
     ASSERT(RuntimeEnabledFeatures::videoTrackEnabled());
 
     // 4.8.10.12.4 Text track API
     // The addTextTrack(kind, label, language) method of media elements, when invoked, must run the following steps:
 
     // 1. If kind is not one of the following strings, then throw a SyntaxError exception and abort these steps
@@ skipping 1288 matching lines @@
         m_fragmentStartTime = MediaPlayer::invalidTime();
 
     double end = fragmentParser.endTime();
     if (end != MediaFragmentURIParser::invalidTimeValue() && end > 0 && end > m_fragmentStartTime) {
         m_fragmentEndTime = end;
         if (m_fragmentEndTime > dur)
             m_fragmentEndTime = dur;
     } else
         m_fragmentEndTime = MediaPlayer::invalidTime();
 
+    // FIXME: Add support for selecting tracks by ID with the Media Fragments track dimension.
+
     if (m_fragmentStartTime != MediaPlayer::invalidTime() && m_readyState < HAVE_FUTURE_DATA)
         prepareToPlay();
 }
 
 void HTMLMediaElement::applyMediaFragmentURI()
 {
     if (m_fragmentStartTime != MediaPlayer::invalidTime()) {
         m_sentEndEvent = false;
         UseCounter::count(document(), UseCounter::HTMLMediaElementSeekToFragmentStart);
         seek(m_fragmentStartTime, IGNORE_EXCEPTION);
@@ skipping 38 matching lines @@
 void HTMLMediaElement::mediaPlayerMediaSourceOpened(blink::WebMediaSource* webMediaSource)
 {
     m_mediaSource->setWebMediaSourceAndOpen(adoptPtr(webMediaSource));
 }
 
 bool HTMLMediaElement::isInteractiveContent() const
 {
     return fastHasAttribute(controlsAttr);
 }
 
+void HTMLMediaElement::createPlaceholderTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Create a placeholder audio track if |m_player| says it has audio but it didn't explicitly announce the tracks.
+    if (m_player->hasAudio() && !audioTracks()->length())
+        addAudioTrack("audio", WebMediaPlayerClient::AudioTrackKindMain, "Audio Track", "", true);
+
+    // Create a placeholder video track if |m_player| says it has video but it didn't explicitly announce the tracks.
+    if (m_player->hasVideo() && !videoTracks()->length())
+        addVideoTrack("video", WebMediaPlayerClient::VideoTrackKindMain, "Video Track", "", true);
 }
+
+void HTMLMediaElement::selectInitialTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Enable the first audio track if an audio track hasn't been enabled yet.
+    if (audioTracks()->length() > 0 && !audioTracks()->hasEnabledTrack())
+        audioTracks()->anonymousIndexedGetter(0)->setEnabled(true);
+
+    // Select the first video track if a video track hasn't been selected yet.
+    if (videoTracks()->length() > 0 && videoTracks()->selectedIndex() == -1)
+        videoTracks()->anonymousIndexedGetter(0)->setSelected(true);
+}
+
+}