Add SourceBuffer implementations of Audio/VideoTracks

third_party/WebKit/Source/modules/mediasource/SourceBuffer.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 30 matching lines @@
 #include "core/fileapi/FileReaderLoader.h"
 #include "core/html/HTMLMediaElement.h"
 #include "core/html/MediaError.h"
 #include "core/html/TimeRanges.h"
 #include "core/html/track/AudioTrack.h"
 #include "core/html/track/AudioTrackList.h"
 #include "core/html/track/TextTrack.h"
 #include "core/html/track/VideoTrack.h"
 #include "core/html/track/VideoTrackList.h"
 #include "core/streams/Stream.h"
+#include "modules/mediasource/AudioTrackSourceBuffer.h"
 #include "modules/mediasource/MediaSource.h"
+#include "modules/mediasource/VideoTrackSourceBuffer.h"
 #include "platform/Logging.h"
 #include "platform/TraceEvent.h"
 #include "public/platform/WebSourceBuffer.h"
 #include "wtf/MathExtras.h"
 
 #include <limits>
 #include <sstream>
 
 using blink::WebSourceBuffer;
 
@@ skipping 163 matching lines @@
     // 6. If the mode attribute equals "sequence", then set the group start timestamp to new timestamp offset.
     if (!m_webSourceBuffer->setTimestampOffset(offset)) {
         MediaSource::logAndThrowDOMException(exceptionState, InvalidStateError, "The timestamp offset may not be set while the SourceBuffer's append state is 'PARSING_MEDIA_SEGMENT'.");
         return;
     }
 
     // 7. Update the attribute to new timestamp offset.
     m_timestampOffset = offset;
 }
 
+AudioTrackList& SourceBuffer::audioTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_audioTracks;
+}
+
+VideoTrackList& SourceBuffer::videoTracks()
+{
+    ASSERT(RuntimeEnabledFeatures::audioVideoTracksEnabled());
+    return *m_videoTracks;
+}
+
 double SourceBuffer::appendWindowStart() const
 {
     return m_appendWindowStart;
 }
 
 void SourceBuffer::setAppendWindowStart(double start, ExceptionState& exceptionState)
 {
     WTF_LOG(Media, "SourceBuffer::setAppendWindowStart %p start=%f", this, start);
     // Section 3.1 appendWindowStart attribute setter steps.
     // https://dvcs.w3.org/hg/html-media/raw-file/tip/media-source/media-source.html#widl-SourceBuffer-appendWindowStart
@@ skipping 254 matching lines @@
         // 6. Set first initialization segment received flag to true.
         m_firstInitializationSegmentReceived = true;
     }
 
     // TODO(servolk): Implement proper MSE init segment received algorithm.
     if (!RuntimeEnabledFeatures::audioVideoTracksEnabled()) {
         m_pendingTracks.clear();
         return;
     }
 
+    if (!m_audioTracks)
+        m_audioTracks = AudioTrackList::create(*m_source->mediaElement());
+    if (!m_videoTracks)
+        m_videoTracks = VideoTrackList::create(*m_source->mediaElement());
+
     for (const auto& trackId : m_audioTrackIds) {
         m_source->mediaElement()->audioTracks().remove(trackId);
+        audioTracks().remove(trackId);

[philipj_slow, 2016/03/10 13:12:17]

This makes it clear that the order in which the track is removed from both track
lists will affect the order of the ends. This doesn't seem to be spelled out by
https://w3c.github.io/media-source/#audio-track-extensions but I could be
looking in the wrong place. This would also need tests.

[servolk, 2016/03/11 02:00:58]

see below

     }
     for (const auto& trackId : m_videoTrackIds) {
         m_source->mediaElement()->videoTracks().remove(trackId);
+        videoTracks().remove(trackId);
     }
     m_audioTrackIds.clear();
     m_videoTrackIds.clear();
 
     for (const auto& trackId : newTracks) {
         for (const auto& track : m_pendingTracks) {
             if (track->trackId() == trackId) {
                 if (track->type() == WebMediaPlayer::AudioTrack) {
                     AudioTrack* audioTrack = static_cast<AudioTrack*>(track.get());
                     WTF_LOG(Media, "Tracks (sb=%p): adding audioTrack %p trackId=%d id=%s label=%s lang=%s", this, audioTrack, audioTrack->trackId(), audioTrack->id().utf8().data(), audioTrack->label().utf8().data(), audioTrack->language().utf8().data());
+                    audioTracks().add(audioTrack);

[philipj_slow, 2016/03/10 13:12:17]

Order tests needed for adding tracks too.

[servolk, 2016/03/11 02:00:58]

I think we are better off postponing these tests until a separate CL that will
implement a fully spec-compliant MSE init segment received algorithm. The
current code removes all the old tracks, but the spec compliant algorithm will
need to check the new track ids and match them with existing tracks, verify that
track properties haven't changed and leave those track in the lists. The MSE
spec does specify the order here, the track must be added to the SB track list
first and then to the HTMLMediaElement (see for example for audio tracks items
5.2.8.8 and 5.2.8.10 at
https://w3c.github.io/media-source/#sourcebuffer-init-segment-received).

                     m_source->mediaElement()->audioTracks().add(audioTrack);
                     m_audioTrackIds.append(trackId);
                 } else if (track->type() == WebMediaPlayer::VideoTrack) {
                     VideoTrack* videoTrack = static_cast<VideoTrack*>(track.get());
                     WTF_LOG(Media, "Tracks (sb=%p): adding videoTrack %p trackId=%d id=%s label=%s lang=%s", this, videoTrack, videoTrack->trackId(), videoTrack->id().utf8().data(), videoTrack->label().utf8().data(), videoTrack->language().utf8().data());
+                    videoTracks().add(videoTrack);
                     m_source->mediaElement()->videoTracks().add(videoTrack);
                     m_videoTrackIds.append(trackId);
                 }
                 break;
             }
         }
     }
     m_pendingTracks.clear();
 }
 
 WebMediaPlayer::TrackId SourceBuffer::createMediaTrack(WebMediaPlayer::TrackType type, WebString id, WebString kind, WebString label, WebString language)
 {
     TrackBase* result = nullptr;
     switch (type) {
     case WebMediaPlayer::AudioTrack:
-        result = AudioTrack::create(id, kind, label, language, false);
-        break;
+        {
+            AudioTrack* audioTrack = AudioTrack::create(id, kind, label, language, false);
+            if (audioTrack) {

[philipj_slow, 2016/03/10 13:12:17]

It can't be null, can it? On OOM we should have crashed already?

[servolk, 2016/03/11 02:00:58]

Done.

+                new AudioTrackSourceBuffer(*audioTrack, this);
+                result = audioTrack;
+            }
+            WTF_LOG(Media, "SourceBuffer(%p)::createMediaTrack: audioTrack %p trackId=%d id=%s label=%s lang=%s", this, audioTrack, audioTrack->trackId(), audioTrack->id().utf8().data(), audioTrack->label().utf8().data(), audioTrack->language().utf8().data());
+            break;
+        }
     case WebMediaPlayer::TextTrack:
         result = TextTrack::create(kind, label, language);
         break;
     case WebMediaPlayer::VideoTrack:
-        result = VideoTrack::create(id, kind, label, language, false);
-        break;
+        {
+            VideoTrack* videoTrack = VideoTrack::create(id, kind, label, language, false);
+            if (videoTrack) {
+                new VideoTrackSourceBuffer(*videoTrack, this);
+                result = videoTrack;
+            }
+            WTF_LOG(Media, "SourceBuffer(%p)::createMediaTrack: videoTrack %p trackId=%d id=%s label=%s lang=%s", this, videoTrack, videoTrack->trackId(), videoTrack->id().utf8().data(), videoTrack->label().utf8().data(), videoTrack->language().utf8().data());
+            break;
+        }
     }
 
     ASSERT(result);
     m_pendingTracks.append(result);
     return result->trackId();
 }
 
 bool SourceBuffer::hasPendingActivity() const
 {
     return m_source;
@@ skipping 368 matching lines @@
     visitor->trace(m_source);
     visitor->trace(m_trackDefaults);
     visitor->trace(m_asyncEventQueue);
     visitor->trace(m_appendBufferAsyncPartRunner);
     visitor->trace(m_removeAsyncPartRunner);
     visitor->trace(m_appendStreamAsyncPartRunner);
     visitor->trace(m_stream);
     visitor->trace(m_pendingTracks);
     visitor->trace(m_audioTrackIds);
     visitor->trace(m_videoTrackIds);
+    visitor->trace(m_audioTracks);
+    visitor->trace(m_videoTracks);
     RefCountedGarbageCollectedEventTargetWithInlineData<SourceBuffer>::trace(visitor);
     ActiveDOMObject::trace(visitor);
 }
 
 } // namespace blink