Add SourceBuffer implementations of Audio/VideoTracks

third_party/WebKit/Source/modules/mediasource/SourceBuffer.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 31 matching lines @@
 #include "core/html/HTMLMediaElement.h"
 #include "core/html/MediaError.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/track/AudioTrack.h"
 #include "core/html/track/AudioTrackList.h"
 #include "core/html/track/TextTrack.h"
 #include "core/html/track/VideoTrack.h"
 #include "core/html/track/VideoTrackList.h"
 #include "core/streams/Stream.h"
 #include "modules/mediasource/MediaSource.h"
+#include "modules/mediasource/SourceBufferTrackBaseSupplement.h"
 #include "platform/Logging.h"
 #include "platform/TraceEvent.h"
 #include "public/platform/WebSourceBuffer.h"
 #include "wtf/MathExtras.h"
 
 #include <limits>
 #include <sstream>
 
 using blink::WebSourceBuffer;
 
@@ skipping 163 matching lines @@
     // 6. If the mode attribute equals "sequence", then set the group start timestamp to new timestamp offset.
     if (!m_webSourceBuffer->setTimestampOffset(offset)) {
         MediaSource::logAndThrowDOMException(exceptionState, InvalidStateError, "The timestamp offset may not be set while the SourceBuffer's append state is 'PARSING_MEDIA_SEGMENT'.");
         return;
     }
 
     // 7. Update the attribute to new timestamp offset.
     m_timestampOffset = offset;
 }
 
+AudioTrackList& SourceBuffer::audioTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_audioTracks;

[wolenetz, 2016/03/23 20:18:52]

Prior to attachment of MediaSource to a MediaElement, it hasn't yet reached open
state (and sourcebuffers cannot be created for the MediaSource). Once it's
attached, (setWebMediaSourceAndOpen()), then AddId() can create SourceBuffers.

So, at creation time, SourceBuffers should know they are part of an 'open'
MediaSource that is attached to a mediaElement.
So why not create these in ctor? (The SourceBuffer ctor is only available via
::create, which is only available via MediaSource.AddId()).

Later, if MediaSource is detached from the element, the spec is a quite a bit
less clear, unfortunately, around what happens to the SB's tracks (and
associated mediaElement tracks). I think the idea would be to do something
similar to the "removeSourceBuffer()" steps as part of detaching from the
element. WDYT? This also needs some testing.

[servolk, 2016/03/31 00:03:40]

Yes, I've changed this already in the previous CL. The track lists are now going
to be created in SB constructor. As for MediaSource being detached from the
element, I think it's covered in MSE spec at
https://w3c.github.io/media-source/#mediasource-detach. Steps 3 and 5 say that
all sourcebuffers should be removed, so yes we'll need to run removeSourceBuffer
and that will take care of media tracks. But you are right in that there seems
to be no test coverage for this atm. I'll try to fix it in a separate CL.

[wolenetz, 2016/03/31 18:57:59]

Acknowledged.

+}
+
+VideoTrackList& SourceBuffer::videoTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_videoTracks;
+}
+
 double SourceBuffer::appendWindowStart() const
 {
     return m_appendWindowStart;
 }
 
 void SourceBuffer::setAppendWindowStart(double start, ExceptionState& exceptionState)
 {
     WTF_LOG(Media, "SourceBuffer::setAppendWindowStart %p start=%f", this, start);
     // Section 3.1 appendWindowStart attribute setter steps.
     // https://dvcs.w3.org/hg/html-media/raw-file/tip/media-source/media-source.html#widl-SourceBuffer-appendWindowStart
@@ skipping 254 matching lines @@
         // 6. Set first initialization segment received flag to true.
         m_firstInitializationSegmentReceived = true;
     }
 
     // TODO(servolk): Implement proper MSE init segment received algorithm.
     if (!RuntimeEnabledFeatures::audioVideoTracksEnabled()) {
         m_pendingTracks.clear();
         return;
     }
 
+    if (!m_audioTracks)
+        m_audioTracks = AudioTrackList::create(*m_source->mediaElement());
+    if (!m_videoTracks)
+        m_videoTracks = VideoTrackList::create(*m_source->mediaElement());
+
     for (const auto& trackId : m_audioTrackIds) {
         m_source->mediaElement()->audioTracks().remove(trackId);
+        audioTracks().remove(trackId);
     }
     for (const auto& trackId : m_videoTrackIds) {
         m_source->mediaElement()->videoTracks().remove(trackId);
+        videoTracks().remove(trackId);
     }
     m_audioTrackIds.clear();
     m_videoTrackIds.clear();
 
     for (const auto& trackId : newTracks) {
         for (const auto& track : m_pendingTracks) {
             if (track->trackId() == trackId) {
                 if (track->type() == WebMediaPlayer::AudioTrack) {
                     AudioTrack* audioTrack = static_cast<AudioTrack*>(track.get());
                     WTF_LOG(Media, "Tracks (sb=%p): adding audioTrack %p trackId=%d id=%s label=%s lang=%s", this, audioTrack, audioTrack->trackId(), audioTrack->id().utf8().data(), audioTrack->label().utf8().data(), audioTrack->language().utf8().data());
+                    audioTracks().add(audioTrack);
                     m_source->mediaElement()->audioTracks().add(audioTrack);
                     m_audioTrackIds.append(trackId);
                 } else if (track->type() == WebMediaPlayer::VideoTrack) {
                     VideoTrack* videoTrack = static_cast<VideoTrack*>(track.get());
                     WTF_LOG(Media, "Tracks (sb=%p): adding videoTrack %p trackId=%d id=%s label=%s lang=%s", this, videoTrack, videoTrack->trackId(), videoTrack->id().utf8().data(), videoTrack->label().utf8().data(), videoTrack->language().utf8().data());
+                    videoTracks().add(videoTrack);
                     m_source->mediaElement()->videoTracks().add(videoTrack);
                     m_videoTrackIds.append(trackId);
                 }
                 break;
             }
         }
     }
     m_pendingTracks.clear();
 }
 
 WebMediaPlayer::TrackId SourceBuffer::createMediaTrack(WebMediaPlayer::TrackType type, WebString id, WebString kind, WebString label, WebString language)
 {
     TrackBase* result = nullptr;
+    const char* typeStr = "";
     switch (type) {
     case WebMediaPlayer::AudioTrack:
         result = AudioTrack::create(id, kind, label, language, false);
+        new SourceBufferTrackBaseSupplement(*result, this);
+        typeStr = "audio";
         break;
     case WebMediaPlayer::TextTrack:
         result = TextTrack::create(kind, label, language);
+        typeStr = "text";
         break;
     case WebMediaPlayer::VideoTrack:
         result = VideoTrack::create(id, kind, label, language, false);
+        new SourceBufferTrackBaseSupplement(*result, this);
+        typeStr = "video";
         break;
     }
+    ASSERT_UNUSED(typeStr, typeStr);
+    WTF_LOG(Media, "SourceBuffer(%p)::createMediaTrack: %s track %p trackId=%d id=%s label=%s lang=%s", this, typeStr, result, result->trackId(), result->id().utf8().data(), result->label().utf8().data(), result->language().utf8().data());
 
     ASSERT(result);
     m_pendingTracks.append(result);
     return result->trackId();
 }
 
 bool SourceBuffer::hasPendingActivity() const
 {
     return m_source;
 }
@@ skipping 367 matching lines @@
     visitor->trace(m_source);
     visitor->trace(m_trackDefaults);
     visitor->trace(m_asyncEventQueue);
     visitor->trace(m_appendBufferAsyncPartRunner);
     visitor->trace(m_removeAsyncPartRunner);
     visitor->trace(m_appendStreamAsyncPartRunner);
     visitor->trace(m_stream);
     visitor->trace(m_pendingTracks);
     visitor->trace(m_audioTrackIds);
     visitor->trace(m_videoTrackIds);
+    visitor->trace(m_audioTracks);
+    visitor->trace(m_videoTracks);
     RefCountedGarbageCollectedEventTargetWithInlineData<SourceBuffer>::trace(visitor);
     ActiveDOMObject::trace(visitor);
 }
 
 } // namespace blink