Initial implementation of AudioTrack, AudioTrackList, VideoTrack, and VideoTrackList.

Source/core/html/HTMLMediaElement.cpp

 /*
  * Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 34 matching lines @@
 #include "core/frame/UseCounter.h"
 #include "core/frame/csp/ContentSecurityPolicy.h"
 #include "core/html/HTMLMediaSource.h"
 #include "core/html/HTMLSourceElement.h"
 #include "core/html/HTMLTrackElement.h"
 #include "core/html/MediaController.h"
 #include "core/html/MediaError.h"
 #include "core/html/MediaFragmentURIParser.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/shadow/MediaControls.h"
+#include "core/html/track/AudioTrack.h"
+#include "core/html/track/AudioTrackList.h"
 #include "core/html/track/InbandTextTrack.h"
 #include "core/html/track/TextTrackCueList.h"
 #include "core/html/track/TextTrackList.h"
+#include "core/html/track/VideoTrack.h"
+#include "core/html/track/VideoTrackList.h"
 #include "core/loader/FrameLoader.h"
 #include "core/rendering/RenderVideo.h"
 #include "core/rendering/RenderView.h"
 #include "core/rendering/compositing/RenderLayerCompositor.h"
 #include "platform/ContentType.h"
 #include "platform/Language.h"
 #include "platform/Logging.h"
 #include "platform/MIMETypeFromURL.h"
 #include "platform/MIMETypeRegistry.h"
 #include "platform/NotImplemented.h"
@@ skipping 10 matching lines @@
 #include "wtf/text/CString.h"
 
 #if ENABLE(WEB_AUDIO)
 #include "platform/audio/AudioSourceProvider.h"
 #include "modules/webaudio/MediaElementAudioSourceNode.h"
 #endif
 
 using namespace std;
 using blink::WebInbandTextTrack;
 using blink::WebMimeRegistry;
+using blink::WebMediaPlayerClient;
 
 namespace WebCore {
 
 #if !LOG_DISABLED
 static String urlForLoggingMedia(const KURL& url)
 {
     static const unsigned maximumURLLengthForLogging = 128;
 
     if (url.string().length() < maximumURLLengthForLogging)
         return url.string();
@@ skipping 58 matching lines @@
     ~TrackDisplayUpdateScope()
     {
         ASSERT(m_mediaElement);
         m_mediaElement->endIgnoringTrackDisplayUpdateRequests();
     }
 
 private:
     HTMLMediaElement* m_mediaElement;
 };
 
+static const AtomicString& VideoKindToString(WebMediaPlayerClient::VideoTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::VideoTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::VideoTrackKindAlternative:
+        return VideoTrack::alternativeKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCaptions:
+        return VideoTrack::captionsKeyword();
+    case WebMediaPlayerClient::VideoTrackKindMain:
+        return VideoTrack::mainKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSign:
+        return VideoTrack::signKeyword();
+    case WebMediaPlayerClient::VideoTrackKindSubtitles:
+        return VideoTrack::subtitlesKeyword();
+    case WebMediaPlayerClient::VideoTrackKindCommentary:
+        return VideoTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
+static const AtomicString& AudioKindToString(WebMediaPlayerClient::AudioTrackKind kind)
+{
+    switch (kind) {
+    case WebMediaPlayerClient::AudioTrackKindNone:
+        return emptyAtom;
+    case WebMediaPlayerClient::AudioTrackKindAlternative:
+        return AudioTrack::alternativeKeyword();
+    case WebMediaPlayerClient::AudioTrackKindDescriptions:
+        return AudioTrack::descriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMain:
+        return AudioTrack::mainKeyword();
+    case WebMediaPlayerClient::AudioTrackKindMainDescriptions:
+        return AudioTrack::mainDescriptionsKeyword();
+    case WebMediaPlayerClient::AudioTrackKindTranslation:
+        return AudioTrack::translationKeyword();
+    case WebMediaPlayerClient::AudioTrackKindCommentary:
+        return AudioTrack::commentaryKeyword();
+    }
+
+    ASSERT_NOT_REACHED();
+    return emptyAtom;
+}
+
 static bool canLoadURL(const KURL& url, const ContentType& contentType, const String& keySystem)
 {
     DEFINE_STATIC_LOCAL(const String, codecs, ("codecs"));
 
     String contentMIMEType = contentType.type().lower();
     String contentTypeCodecs = contentType.parameter(codecs);
 
     // If the MIME type is missing or is not meaningful, try to figure it out from the URL.
     if (contentMIMEType.isEmpty() || contentMIMEType == "application/octet-stream" || contentMIMEType == "text/plain") {
         if (url.protocolIsData())
@@ skipping 126 matching lines @@
 {
     WTF_LOG(Media, "HTMLMediaElement::~HTMLMediaElement");
 
     m_asyncEventQueue->close();
 
     setShouldDelayLoadEvent(false);
 
     if (m_textTracks)
         m_textTracks->clearOwner();
 
+    if (m_audioTracks)
+        m_audioTracks->shutdown();
+
+    if (m_videoTracks)
+        m_videoTracks->shutdown();
+
     if (m_mediaController) {
         m_mediaController->removeMediaElement(this);
         m_mediaController = nullptr;
     }
 
     closeMediaSource();
 
     removeElementFromDocumentMap(this, &document());
 
     // Destroying the player may cause a resource load to be canceled,
@@ skipping 39 matching lines @@
     // previous document. A proper fix would provide a mechanism to allow this
     // object to refresh the MediaPlayer's LocalFrame and FrameLoader references on
     // document changes so that playback can be resumed properly.
     userCancelledLoad();
 
     // Decrement the load event delay count on oldDocument now that m_player has been destroyed
     // and there is no risk of dispatching a load event from within the destructor.
     oldDocument.decrementLoadEventDelayCount();
 
     ActiveDOMObject::didMoveToNewExecutionContext(&document());
+
+    // Notify track lists so they don't stop firing events when moved to a new document
+    // and the original document gets destroyed.
+    if (m_audioTracks)
+        m_audioTracks->didMoveToNewExecutionContext(&document());
+
+    if (m_videoTracks)
+        m_videoTracks->didMoveToNewExecutionContext(&document());
+
     HTMLElement::didMoveToNewDocument(oldDocument);
 }
 
 bool HTMLMediaElement::hasCustomFocusLogic() const
 {
     return true;
 }
 
 bool HTMLMediaElement::supportsFocus() const
 {
@@ skipping 1178 matching lines @@
             finishSeek();
     } else {
         if (wasPotentiallyPlaying && m_readyState < HAVE_FUTURE_DATA) {
             // 4.8.10.8
             scheduleTimeupdateEvent(false);
             scheduleEvent(EventTypeNames::waiting);
         }
     }
 
     if (m_readyState >= HAVE_METADATA && oldState < HAVE_METADATA) {
+        createPlaceholderTracksIfNecessary();
+
         prepareMediaFragmentURI();
+
+        selectInitialTracksIfNecessary();
+
         scheduleEvent(EventTypeNames::durationchange);
         if (isVideo())
             scheduleEvent(EventTypeNames::resize);
         scheduleEvent(EventTypeNames::loadedmetadata);
         if (hasMediaControls())
             mediaControls()->reset();
         if (renderer())
             renderer()->updateFromElement();
     }
 
@@ skipping 602 matching lines @@
         m_lastTimeUpdateEventWallTime = now;
         m_lastTimeUpdateEventMovieTime = movieTime;
     }
 }
 
 bool HTMLMediaElement::canPlay() const
 {
     return paused() || ended() || m_readyState < HAVE_METADATA;
 }
 
+AudioTrackList& HTMLMediaElement::audioTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    if (!m_audioTracks)
+        m_audioTracks = AudioTrackList::create(this);
+
+    return *m_audioTracks;
+}
+
+void HTMLMediaElement::audioTrackChanged(const AtomicString& audioTrackID, bool enabled)
+{
+    WTF_LOG(Media, "HTMLMediaElement::audioTrackChanged('%s', %d)", audioTrackID.ascii().data(), enabled);
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    ASSERT(!audioTrackID.isEmpty());
+
+    audioTracks().scheduleChangeEvent();
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.audioTracks attribute is added.
+
+    if (webMediaPlayer())
+        webMediaPlayer()->enabledAudioTrackChange(audioTrackID, enabled);
+}
+
+VideoTrackList& HTMLMediaElement::videoTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    if (!m_videoTracks)
+        m_videoTracks = VideoTrackList::create(this);
+
+    return *m_videoTracks;
+}
+
+void HTMLMediaElement::selectedVideoTrackChanged(const AtomicString& selectedTrackID)
+{
+    WTF_LOG(Media, "HTMLMediaElement::selectedVideoTrackChanged('%s')", selectedTrackID.ascii().data());
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+
+    AtomicString unselectedTrackID;
+
+    int oldSelectedIndex = videoTracks().selectedIndex();
+    if (oldSelectedIndex != -1)
+        unselectedTrackID = videoTracks().anonymousIndexedGetter(oldSelectedIndex)->id();
+
+    ASSERT(unselectedTrackID != selectedTrackID);
+
+    videoTracks().trackSelected(selectedTrackID);
+
+    // FIXME: Add call on m_mediaSource to notify it of track changes once the SourceBuffer.videoTracks attribute is added.
+
+    ASSERT(!unselectedTrackID.isEmpty() || !selectedTrackID.isEmpty());
+
+    if (webMediaPlayer())
+        webMediaPlayer()->selectedVideoTrackChange(unselectedTrackID, selectedTrackID);
+}
+
+void HTMLMediaElement::addVideoTrack(const AtomicString& id, blink::WebMediaPlayerClient::VideoTrackKind kind, const AtomicString& label, const AtomicString& language, bool selected)
+{
+    AtomicString kindString = VideoKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addVideoTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), selected);
+    ASSERT(!id.isEmpty());
+
+    RefPtr<VideoTrack> videoTrack = VideoTrack::create(this, id, kindString, label, language);
+    videoTracks().add(videoTrack.get());
+
+    if (selected)
+        videoTrack->setSelected(true);
+}
+
+void HTMLMediaElement::removeVideoTrack(const AtomicString& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeVideoTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    videoTracks().remove(id);
+}
+
+void HTMLMediaElement::addAudioTrack(const AtomicString& id, blink::WebMediaPlayerClient::AudioTrackKind kind, const AtomicString& label, const AtomicString& language, bool enabled)
+{
+    AtomicString kindString = AudioKindToString(kind);
+    WTF_LOG(Media, "HTMLMediaElement::addAudioTrack('%s', '%s', '%s', '%s', %d)",
+        id.ascii().data(), kindString.ascii().data(), label.ascii().data(), language.ascii().data(), enabled);
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    RefPtr<AudioTrack> audioTrack = AudioTrack::create(this, id, kindString, label, language);
+    audioTracks().add(audioTrack.get());
+
+    if (enabled)
+        audioTrack->setEnabled(true);
+}
+
+void HTMLMediaElement::removeAudioTrack(const AtomicString& id)
+{
+    WTF_LOG(Media, "HTMLMediaElement::removeAudioTrack('%s')", id.ascii().data());
+    ASSERT(!id.isEmpty());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    audioTracks().remove(id);
+}
+
 void HTMLMediaElement::mediaPlayerDidAddTextTrack(WebInbandTextTrack* webTrack)
 {
     if (!RuntimeEnabledFeatures::videoTrackEnabled())
         return;
 
     // 4.8.10.12.2 Sourcing in-band text tracks
     // 1. Associate the relevant data with a new text track and its corresponding new TextTrack object.
     RefPtr<InbandTextTrack> textTrack = InbandTextTrack::create(document(), webTrack);
 
     // 2. Set the new text track's kind, label, and language based on the semantics of the relevant data,
@@ skipping 55 matching lines @@
 void HTMLMediaElement::removeTextTrack(TextTrack* track)
 {
     TrackDisplayUpdateScope scope(this);
     m_textTracks->remove(track);
 
     closeCaptionTracksChanged();
 }
 
 void HTMLMediaElement::forgetResourceSpecificTracks()
 {
+    // Implements the "forget the media element's media-resource-specific tracks" algorithm.
+    // The order is explicitly specified as text, then audio, and finally video. Also
+    // 'removetrack' events should not be fired.
     if (m_textTracks) {
         TrackDisplayUpdateScope scope(this);
         m_textTracks->removeAllInbandTracks();
         closeCaptionTracksChanged();
     }
+
+    if (m_audioTracks)
+        m_audioTracks->removeAll();
+
+    if (m_videoTracks)
+        m_videoTracks->removeAll();
 }
 
 PassRefPtr<TextTrack> HTMLMediaElement::addTextTrack(const AtomicString& kind, const AtomicString& label, const AtomicString& language, ExceptionState& exceptionState)
 {
     ASSERT(RuntimeEnabledFeatures::videoTrackEnabled());
 
     // 4.8.10.12.4 Text track API
     // The addTextTrack(kind, label, language) method of media elements, when invoked, must run the following steps:
 
     // 1. If kind is not one of the following strings, then throw a SyntaxError exception and abort these steps
@@ skipping 1291 matching lines @@
         m_fragmentStartTime = MediaPlayer::invalidTime();
 
     double end = fragmentParser.endTime();
     if (end != MediaFragmentURIParser::invalidTimeValue() && end > 0 && end > m_fragmentStartTime) {
         m_fragmentEndTime = end;
         if (m_fragmentEndTime > dur)
             m_fragmentEndTime = dur;
     } else
         m_fragmentEndTime = MediaPlayer::invalidTime();
 
+    // FIXME: Add support for selecting tracks by ID with the Media Fragments track dimension.
+
     if (m_fragmentStartTime != MediaPlayer::invalidTime() && m_readyState < HAVE_FUTURE_DATA)
         prepareToPlay();
 }
 
 void HTMLMediaElement::applyMediaFragmentURI()
 {
     if (m_fragmentStartTime != MediaPlayer::invalidTime()) {
         m_sentEndEvent = false;
         UseCounter::count(document(), UseCounter::HTMLMediaElementSeekToFragmentStart);
         seek(m_fragmentStartTime, IGNORE_EXCEPTION);
@@ skipping 38 matching lines @@
 void HTMLMediaElement::mediaPlayerMediaSourceOpened(blink::WebMediaSource* webMediaSource)
 {
     m_mediaSource->setWebMediaSourceAndOpen(adoptPtr(webMediaSource));
 }
 
 bool HTMLMediaElement::isInteractiveContent() const
 {
     return fastHasAttribute(controlsAttr);
 }
 
+void HTMLMediaElement::createPlaceholderTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Create a placeholder audio track if |m_player| says it has audio but it didn't explicitly announce the tracks.
+    if (m_player->hasAudio() && !audioTracks().length())
+        addAudioTrack("audio", WebMediaPlayerClient::AudioTrackKindMain, "Audio Track", "", true);
+
+    // Create a placeholder video track if |m_player| says it has video but it didn't explicitly announce the tracks.
+    if (m_player->hasVideo() && !videoTracks().length())

[philipj_slow, 2014/03/14 03:51:59]

hasVideo() can only be true for HTMLVideoElement, but the spec's "If the media
resource is found to have a video track" steps don't discriminate between audio
and video elements. In other words, we need to test that audio elements work the
same as video elements as far as this is concerned. We could also file a spec
bug, but I'm not sure hiding video tracks for HTMLAudioElement would actually be
useful.

[acolwell GONE FROM CHROMIUM, 2014/03/18 22:02:15]

This is m_player->hasVideo() not hasVideo(). It reports the presence of video
tracks independent of whether this is an HTMLVideoElement or HTMLAudioElement.

[philipj_slow, 2014/03/20 16:17:40]

You're right.

+        addVideoTrack("video", WebMediaPlayerClient::VideoTrackKindMain, "Video Track", "", true);
 }
+
+void HTMLMediaElement::selectInitialTracksIfNecessary()
+{
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled())
+        return;
+
+    // Enable the first audio track if an audio track hasn't been enabled yet.
+    if (audioTracks().length() > 0 && !audioTracks().hasEnabledTrack())
+        audioTracks().anonymousIndexedGetter(0)->setEnabled(true);
+
+    // Select the first video track if a video track hasn't been selected yet.
+    if (videoTracks().length() > 0 && videoTracks().selectedIndex() == -1)
+        videoTracks().anonymousIndexedGetter(0)->setSelected(true);
+}
+
+}