Pass MSE media track info from ChunkDemuxer to blink::SourceBuffer

third_party/WebKit/Source/modules/mediasource/SourceBuffer.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 24 matching lines @@
 #include "core/dom/DOMArrayBuffer.h"
 #include "core/dom/DOMArrayBufferView.h"
 #include "core/dom/ExceptionCode.h"
 #include "core/dom/ExecutionContext.h"
 #include "core/events/Event.h"
 #include "core/events/GenericEventQueue.h"
 #include "core/fileapi/FileReaderLoader.h"
 #include "core/html/HTMLMediaElement.h"
 #include "core/html/MediaError.h"
 #include "core/html/TimeRanges.h"
+#include "core/html/track/AudioTrack.h"
+#include "core/html/track/AudioTrackList.h"
+#include "core/html/track/TextTrack.h"
+#include "core/html/track/VideoTrack.h"
+#include "core/html/track/VideoTrackList.h"
 #include "core/streams/Stream.h"
 #include "modules/mediasource/MediaSource.h"
 #include "platform/Logging.h"
 #include "platform/TraceEvent.h"
 #include "public/platform/WebSourceBuffer.h"
 #include "wtf/MathExtras.h"
 
 #include <limits>
 #include <sstream>
 
@@ skipping 408 matching lines @@
 
     WTF_LOG(Media, "SourceBuffer(%p)::removedFromMediaSource", this);
     abortIfUpdating();
 
     m_webSourceBuffer->removedFromMediaSource();
     m_webSourceBuffer.clear();
     m_source = nullptr;
     m_asyncEventQueue = nullptr;
 }
 
-void SourceBuffer::initializationSegmentReceived()
+void SourceBuffer::initializationSegmentReceived(const WebVector<WebMediaPlayer::TrackId>& newTracks)
 {
-    WTF_LOG(Media, "SourceBuffer::initializationSegmentReceived %p", this);
+    WTF_LOG(Media, "SourceBuffer::initializationSegmentReceived %p tracks=%zu", this, newTracks.size());
     ASSERT(m_source);
+    ASSERT(m_source->mediaElement());
     ASSERT(m_updating);
+    ASSERT(newTracks.size() == m_pendingTracks.size());
 
     // https://dvcs.w3.org/hg/html-media/raw-file/tip/media-source/media-source.html#sourcebuffer-init-segment-received
     // FIXME: Make steps 1-7 synchronous with this call.
     // FIXME: Augment the interface to this method to implement compliant steps 4-7 here.
     // Step 3 (if the first initialization segment received flag is true) is
     // implemented by caller.
 
     if (!m_firstInitializationSegmentReceived) {
         // 5. If active track flag equals true, then run the following steps:
         // 5.1. Add this SourceBuffer to activeSourceBuffers.
         // 5.2. Queue a task to fire a simple event named addsourcebuffer at
         // activesourcebuffers.
         m_source->setSourceBufferActive(this);
 
         // 6. Set first initialization segment received flag to true.
         m_firstInitializationSegmentReceived = true;
     }
+
+    // TODO(servolk): Implement proper MSE init segment received algorithm.
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled()) {
+        m_pendingTracks.clear();
+        return;
+    }
+
+    for (const auto& trackId : m_audioTrackIds) {
+        m_source->mediaElement()->audioTracks().remove(trackId);

[philipj_slow, 2016/03/24 05:56:00]

Can't this end up adding and removing the same track, firing spurious events?
Unless I'm misunderstanding things still, the eventual shape of things should be
that we get some tracks with the first call to initializationSegmentReceived,
and they can never change after that because that should have been an error. So
could you assert that there are no existing tracks? And maybe an early return
somewhere above if this isn't the first init segment?

(I'm not sure if initializationSegmentReceived is going to end up called
multiple times or not, or if it's the responsibility of this code to handle the
situation.)

[servolk, 2016/03/29 01:26:33]

initializationSegmentReceived is definitely going to be invoked multiple times,
it gets invoked every time ChunkDemuxer receives a new init segment, which might
happen in the middle of playback, for example if the player decides to switch to
a different bitrate/quality level. And you are right, that in the previous
patchsets it would cause multiple change events to be fired. But now I've
reworked it a bit more (+ see below) and in the latest patchset of this CL that
will no longer happen. We will add media tracks only once, and then subsequent
init segments should be able to reuse existing tracks.

+    }
+    for (const auto& trackId : m_videoTrackIds) {
+        m_source->mediaElement()->videoTracks().remove(trackId);
+    }
+    m_audioTrackIds.clear();
+    m_videoTrackIds.clear();
+
+    for (const auto& trackId : newTracks) {
+        for (const auto& track : m_pendingTracks) {
+            if (track->trackId() == trackId) {
+                if (track->type() == WebMediaPlayer::AudioTrack) {
+                    AudioTrack* audioTrack = static_cast<AudioTrack*>(track.get());
+                    WTF_LOG(Media, "Tracks (sb=%p): adding audioTrack %p trackId=%d id=%s label=%s lang=%s", this, audioTrack, audioTrack->trackId(), audioTrack->id().utf8().data(), audioTrack->label().utf8().data(), audioTrack->language().utf8().data());
+                    m_source->mediaElement()->audioTracks().add(audioTrack);
+                    m_audioTrackIds.append(trackId);
+                } else if (track->type() == WebMediaPlayer::VideoTrack) {
+                    VideoTrack* videoTrack = static_cast<VideoTrack*>(track.get());
+                    WTF_LOG(Media, "Tracks (sb=%p): adding videoTrack %p trackId=%d id=%s label=%s lang=%s", this, videoTrack, videoTrack->trackId(), videoTrack->id().utf8().data(), videoTrack->label().utf8().data(), videoTrack->language().utf8().data());
+                    m_source->mediaElement()->videoTracks().add(videoTrack);
+                    m_videoTrackIds.append(trackId);
+                }
+                break;
+            }
+        }
+    }
+    m_pendingTracks.clear();
+}
+
+WebMediaPlayer::TrackId SourceBuffer::createMediaTrack(WebMediaPlayer::TrackType type, WebString id, WebString kind, WebString label, WebString language)
+{
+    TrackBase* result = nullptr;
+    switch (type) {
+    case WebMediaPlayer::AudioTrack:
+        result = AudioTrack::create(id, kind, label, language, false);
+        break;
+    case WebMediaPlayer::TextTrack:
+        result = TextTrack::create(kind, label, language);
+        break;
+    case WebMediaPlayer::VideoTrack:
+        result = VideoTrack::create(id, kind, label, language, false);
+        break;
+    }
+
+    ASSERT(result);
+    m_pendingTracks.append(result);

[philipj_slow, 2016/03/24 05:56:00]

These only end up used in initializationSegmentReceived, where they are
referenced by id. For an API that's more obvious in what it does, have you
looked into simply passing a vector of the actual track data to
initializationSegmentReceived, so that the tracks are created there? A
WebVector<std::tuple<type, id, kind, label, language>> should do it, probably no
need for a new struct or class.

The track IDs aren't used on the Chromium side now, but if they are later
needed, a vector of assigned track IDs could be returned. This *looks* like a
simplification to me, if it actually complicates matters please clue me in :)

[servolk, 2016/03/29 01:26:33]

Yes, the same functionality can be achieved in a slightly different way. I have
reworked this CL and the latest patchset implements your suggestion (passing
track info as a bunch of std::tuples and returning blink track ids as a vector).
The track IDs are not used on Chromium side in this CL, but that is going to
change very soon. See the subsequent CLs that depend on this one, they required
some minor updates after reworking this CL, of course, but I've rebased and
uploaded latest versions of all the CLs. The track control CL
(https://codereview.chromium.org/1812543003/) is using track IDs to allow
muting/unmuting audio through Javascript MediaTrack APIs.

+    return result->trackId();
 }
 
 bool SourceBuffer::hasPendingActivity() const
 {
     return m_source;
 }
 
 void SourceBuffer::suspend()
 {
     m_appendBufferAsyncPartRunner->suspend();
@@ skipping 360 matching lines @@
 
 DEFINE_TRACE(SourceBuffer)
 {
     visitor->trace(m_source);
     visitor->trace(m_trackDefaults);
     visitor->trace(m_asyncEventQueue);
     visitor->trace(m_appendBufferAsyncPartRunner);
     visitor->trace(m_removeAsyncPartRunner);
     visitor->trace(m_appendStreamAsyncPartRunner);
     visitor->trace(m_stream);
+    visitor->trace(m_pendingTracks);
+    visitor->trace(m_audioTrackIds);
+    visitor->trace(m_videoTrackIds);
     RefCountedGarbageCollectedEventTargetWithInlineData<SourceBuffer>::trace(visitor);
     ActiveDOMObject::trace(visitor);
 }
 
 } // namespace blink