Implement InitSegmentReceived algorithm in blink

third_party/WebKit/Source/modules/mediasource/SourceBuffer.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 484 matching lines @@
 {
     // According to MSE specification (https://w3c.github.io/media-source/#sourcebuffer-init-segment-received) step 3.1:
     // > If more than one track for a single type are present (ie 2 audio tracks), then the Track IDs match the ones in the first initialization segment.
     // I.e. we only need to search by TrackID if there is more than one track, otherwise we can assume that the only
     // track of the given type is the same one that we had in previous init segments.
     if (trackList.length() == 1)
         return trackList.anonymousIndexedGetter(0);
     return trackList.getTrackById(id);
 }
 
+const TrackDefault* SourceBuffer::getTrackDefaultHelper(const AtomicString& trackType, const AtomicString& byteStreamTrackID) const

[philipj_slow, 2016/04/25 13:03:42]

Maybe just getTrackDefault as the name?

[servolk, 2016/04/25 20:29:07]

Done.

+{
+    // This is a helper for implementation of default track label and default track language algorithms.
+    // defaultTrackLabel spec: https://w3c.github.io/media-source/#sourcebuffer-default-track-label
+    // defaultTrackLanguage spec: https://w3c.github.io/media-source/#sourcebuffer-default-track-language
+
+    // 1. If trackDefaults contains a TrackDefault object with a type attribute equal to type and a byteStreamTrackID attribute equal to byteStreamTrackID,
+    // then return the value of the label/language attribute on this matching object and abort these steps.
+    for (unsigned i = 0; i < m_trackDefaults->length(); ++i) {
+        const auto* trackDefault = m_trackDefaults->item(i);

[philipj_slow, 2016/04/25 13:03:42]

From http://chromium-cpp.appspot.com and
https://google.github.io/styleguide/cppguide.html#auto I think this ought to be
just const TrackDefault*.

[servolk, 2016/04/25 20:29:08]

Done.

+        ASSERT(trackDefault);

[philipj_slow, 2016/04/25 13:03:42]

Not sure if there's a document rule here, but in
https://groups.google.com/a/chromium.org/forum/#!msg/chromium-dev/ti040zXE_A8...
most who weigh in don't think we should DCHECK immediately before dereferencing,
and it generally isn't done at least in Blink.

[servolk, 2016/04/25 20:29:08]

Done.

+        if (trackDefault->type() == trackType && trackDefault->byteStreamTrackID() == byteStreamTrackID)
+            return trackDefault;
+    }
+
+    // 2. If trackDefaults contains a TrackDefault object with a type attribute equal to type and a byteStreamTrackID attribute equal to an empty string,
+    // then return the value of the label/language attribute on this matching object and abort these steps.
+    for (unsigned i = 0; i < m_trackDefaults->length(); ++i) {
+        const auto* trackDefault = m_trackDefaults->item(i);
+        ASSERT(trackDefault);
+        if (trackDefault->type() == trackType && trackDefault->byteStreamTrackID() == "")

[philipj_slow, 2016/04/25 13:03:42]

In the first loop you could maintain two variables and then just return one of
those to avoid the second loop.

[servolk, 2016/04/25 20:29:08]

Done.

+            return trackDefault;
+    }
+
+    // 3. Return an empty string to the caller
+    return nullptr;
+}
+
+AtomicString SourceBuffer::defaultTrackLabel(const AtomicString& trackType, const AtomicString& byteStreamTrackID) const
+{
+    // Spec: https://w3c.github.io/media-source/#sourcebuffer-default-track-label
+    const TrackDefault* trackDefault = getTrackDefaultHelper(trackType, byteStreamTrackID);
+    return trackDefault ? AtomicString(trackDefault->label()) : "";
+}
+
+AtomicString SourceBuffer::defaultTrackLanguage(const AtomicString& trackType, const AtomicString& byteStreamTrackID) const
+{
+    // Spec: https://w3c.github.io/media-source/#sourcebuffer-default-track-language
+    const TrackDefault* trackDefault = getTrackDefaultHelper(trackType, byteStreamTrackID);
+    return trackDefault ? AtomicString(trackDefault->language()) : "";
+}
+
 WebVector<WebMediaPlayer::TrackId> SourceBuffer::initializationSegmentReceived(const WebVector<MediaTrackInfo>& newTracks)
 {
     WTF_LOG(Media, "SourceBuffer::initializationSegmentReceived %p tracks=%zu", this, newTracks.size());
     ASSERT(m_source);
     ASSERT(m_source->mediaElement());
     ASSERT(m_updating);
 
-    // TODO(servolk): Implement proper 'initialization segment received' algorithm according to MSE spec:
+    WebVector<WebMediaPlayer::TrackId> assignedTrackIds(newTracks.size());
+
+    if (!RuntimeEnabledFeatures::audioVideoTracksEnabled()) {
+        for (size_t i = 0; i < newTracks.size(); ++i) {
+            static WebMediaPlayer::TrackId nextTrackId = 0;
+            assignedTrackIds[i] = ++nextTrackId;
+        }
+        if (!m_firstInitializationSegmentReceived) {
+            m_source->setSourceBufferActive(this);
+            m_firstInitializationSegmentReceived = true;
+        }
+        return assignedTrackIds;
+    }
+
+
+    // Implementation of Initialization Segment Received, see 3.5.8 at
     // https://w3c.github.io/media-source/#sourcebuffer-init-segment-received
-    WebVector<WebMediaPlayer::TrackId> result(newTracks.size());
+
+    // Sort newTracks into audio and video tracks to facilitate implementation
+    // of subsequent steps of this algorithm.
+    std::vector<MediaTrackInfo> newAudioTracks;

[philipj_slow, 2016/04/25 13:03:42]

Vector<>

[servolk, 2016/04/25 20:29:08]

Done.

+    std::vector<MediaTrackInfo> newVideoTracks;
     unsigned resultIdx = 0;

[philipj_slow, 2016/04/25 13:03:42]

size_t to match WebVector.h and the new usage above?

[servolk, 2016/04/25 20:29:08]

Done.

     for (const auto& trackInfo : newTracks) {
-        if (!RuntimeEnabledFeatures::audioVideoTracksEnabled()) {
-            static WebMediaPlayer::TrackId nextTrackId = 0;
-            result[resultIdx++] = ++nextTrackId;
-            continue;
+        if (trackInfo.trackType == WebMediaPlayer::AudioTrack) {
+            newAudioTracks.push_back(trackInfo);
+            if (m_firstInitializationSegmentReceived) {
+                AudioTrack* audioTrack = findExistingTrackById(audioTracks(), trackInfo.byteStreamTrackId);
+                ASSERT(audioTrack);

[philipj_slow, 2016/04/25 13:03:42]

It seems a bit odd to assume the tracks are all there when the
tracksMatchFirstInitSegment are all about checking that things add up. Is it
really guaranteed that there will be an existing track with the same ID, no
matter how one uses the web-exposed API?

[servolk, 2016/04/25 20:29:07]

Yeah, I guess strictly speaking it's better to handle that the same as
mismatching track ids in subsequent init segments.

+                assignedTrackIds[resultIdx++] = audioTrack->trackId();
+            }
+        } else if (trackInfo.trackType == WebMediaPlayer::VideoTrack) {
+            newVideoTracks.push_back(trackInfo);
+            if (m_firstInitializationSegmentReceived) {
+                VideoTrack* videoTrack = findExistingTrackById(videoTracks(), trackInfo.byteStreamTrackId);
+                ASSERT(videoTrack);
+                assignedTrackIds[resultIdx++] = videoTrack->trackId();
+            }
+        } else {
+            // TODO(servolk): Add handling of text tracks.
+            ASSERT_NOT_REACHED();

[philipj_slow, 2016/04/25 13:03:42]

NOTREACHED() is the new ASSERT_NOT_REACHED()

[servolk, 2016/04/25 20:29:08]

Done.

+        }
+    }
+
+    // 1. Update the duration attribute if it currently equals NaN:
+    // TODO(servolk): Pass also stream duration into initSegmentReceived.
+
+    // 2. If the initialization segment has no audio, video, or text tracks, then run the append error algorithm with the decode error parameter set to true and abort these steps.
+    if (newTracks.size() == 0) {
+        WTF_LOG(Media, "SourceBuffer(%p)::initializationSegmentReceived failed, no tracks found in the init segment.", this);
+        appendError(true);
+        return assignedTrackIds;
+    }
+
+    // 3. If the first initialization segment received flag is true, then run the following steps:
+    if (m_firstInitializationSegmentReceived) {
+        // 3.1 Verify the following properties. If any of the checks fail then run the append error algorithm with the decode error parameter set to true and abort these steps.
+        bool tracksMatchFirstInitSegment = true;
+        // - The number of audio, video, and text tracks match what was in the first initialization segment.
+        if (newAudioTracks.size() != audioTracks().length() || newVideoTracks.size() != videoTracks().length()) {
+            tracksMatchFirstInitSegment = false;
+        }
+        // - The codecs for each track, match what was specified in the first initialization segment.

[philipj_slow, 2016/04/25 13:03:42]

Is this also in MediaSourceState::OnNewConfigs? If so can you add a comment
here?

[servolk, 2016/04/25 20:29:08]

Done.

+        // - If more than one track for a single type are present (ie 2 audio tracks), then the Track IDs match the ones in the first initialization segment.
+        if (tracksMatchFirstInitSegment && newAudioTracks.size() > 1) {
+            for (size_t i = 0; i < newAudioTracks.size(); ++i) {
+                const String& newTrackId = newVideoTracks[i].byteStreamTrackId;
+                if (newTrackId != audioTracks().anonymousIndexedGetter(i)->id())
+                    tracksMatchFirstInitSegment = false;

[philipj_slow, 2016/04/25 13:03:42]

Also break?

[servolk, 2016/04/25 20:29:08]

Done.

+            }
         }
 
-        const TrackBase* trackBase = nullptr;
-        if (trackInfo.trackType == WebMediaPlayer::AudioTrack) {
-            AudioTrack* audioTrack = nullptr;
-            if (!m_firstInitializationSegmentReceived) {
-                audioTrack = AudioTrack::create(trackInfo.byteStreamTrackId, trackInfo.kind, trackInfo.label, trackInfo.language, false);
-                SourceBufferTrackBaseSupplement::setSourceBuffer(*audioTrack, this);
-                audioTracks().add(audioTrack);
-                m_source->mediaElement()->audioTracks().add(audioTrack);
-            } else {
-                audioTrack = findExistingTrackById(audioTracks(), trackInfo.byteStreamTrackId);
-                ASSERT(audioTrack);
+        if (tracksMatchFirstInitSegment && newVideoTracks.size() > 1) {
+            for (size_t i = 0; i < newVideoTracks.size(); ++i) {
+                const String& newTrackId = newVideoTracks[i].byteStreamTrackId;
+                if (newTrackId != videoTracks().anonymousIndexedGetter(i)->id())
+                    tracksMatchFirstInitSegment = false;

[philipj_slow, 2016/04/25 13:03:42]

Ditto.

[servolk, 2016/04/25 20:29:08]

Done.

             }
-            trackBase = audioTrack;
-            result[resultIdx++] = audioTrack->trackId();
-        } else if (trackInfo.trackType == WebMediaPlayer::VideoTrack) {
-            VideoTrack* videoTrack = nullptr;
-            if (!m_firstInitializationSegmentReceived) {
-                videoTrack = VideoTrack::create(trackInfo.byteStreamTrackId, trackInfo.kind, trackInfo.label, trackInfo.language, false);
-                SourceBufferTrackBaseSupplement::setSourceBuffer(*videoTrack, this);
-                videoTracks().add(videoTrack);
-                m_source->mediaElement()->videoTracks().add(videoTrack);
-            } else {
-                videoTrack = findExistingTrackById(videoTracks(), trackInfo.byteStreamTrackId);
-                ASSERT(videoTrack);
-            }
-            trackBase = videoTrack;
-            result[resultIdx++] = videoTrack->trackId();
-        } else {
-            NOTREACHED();
         }
-        (void)trackBase;
-#if !LOG_DISABLED
-        const char* logActionStr = m_firstInitializationSegmentReceived ? "using existing" : "added";
-        const char* logTrackTypeStr = (trackInfo.trackType == WebMediaPlayer::AudioTrack) ? "audio" : "video";
-        WTF_LOG(Media, "Tracks (sb=%p): %s %sTrack %p trackId=%d id=%s label=%s lang=%s", this, logActionStr, logTrackTypeStr, trackBase, trackBase->trackId(), trackBase->id().utf8().data(), trackBase->label().utf8().data(), trackBase->language().utf8().data());
-#endif
+
+        if (!tracksMatchFirstInitSegment) {
+            WTF_LOG(Media, "SourceBuffer(%p)::initializationSegmentReceived failed: tracks mismatch the first init segment.", this);
+            appendError(true);
+            return assignedTrackIds;
+        }
+
+        // 3.2 Add the appropriate track descriptions from this initialization segment to each of the track buffers.
+        // This is done in Chromium code in stream parsers and demuxer implementations.
+
+        // 3.3 Set the need random access point flag on all track buffers to true.
+        // This is done in Chromium code, see MediaSourceState::OnNewConfigs.
     }
 
+    // 4. Let active track flag equal false.
+    m_activeTrack = false;
+
+    // 5. If the first initialization segment received flag is false, then run the following steps:
     if (!m_firstInitializationSegmentReceived) {
-        // 5. If active track flag equals true, then run the following steps:
-        // 5.1. Add this SourceBuffer to activeSourceBuffers.
-        // 5.2. Queue a task to fire a simple event named addsourcebuffer at
+        // 5.1 If the initialization segment contains tracks with codecs the user agent does not support, then run the append error algorithm with the decode error parameter set to true and abort these steps.
+        // This is done in Chromium code, see MediaSourceState::OnNewConfigs.
+
+        // 5.2 For each audio track in the initialization segment, run following steps:
+        for (const auto& trackInfo : newAudioTracks) {

[philipj_slow, 2016/04/25 13:03:42]

s/auto/MediaTrackInfo/

[servolk, 2016/04/25 20:29:08]

Done.

+            // 5.2.1 Let audio byte stream track ID be the Track ID for the current track being processed.
+            const auto& byteStreamTrackId = trackInfo.byteStreamTrackId;

[philipj_slow, 2016/04/25 13:03:42]

s/auto/WebString/ and I think you could just use const WebString& for the locals
below as well. At any rate using the same type for these seems sensible.

[servolk, 2016/04/25 20:29:08]

Done (using WebString). We can't use const reference, because we need to
reassign it, in case we need to use the value from TrackDefaults.

+            // 5.2.2 Let audio language be a BCP 47 language tag for the language specified in the initialization segment for this track or an empty string if no language info is present.
+            AtomicString language = trackInfo.language;
+            // 5.2.3 If audio language equals an empty string or the 'und' BCP 47 value, then run the default track language algorithm with byteStreamTrackID set to
+            // audio byte stream track ID and type set to "audio" and assign the value returned by the algorithm to audio language.
+            if (language == "" || language == "und")

[philipj_slow, 2016/04/25 13:03:42]

language.isEmpty() would also cover the case where language.isNull() is true, if
that can ever happen.

[servolk, 2016/04/25 20:29:08]

Done.

+                language = defaultTrackLanguage("audio", byteStreamTrackId);

[philipj_slow, 2016/04/25 13:03:42]

Can you expose audioKeyword() from TrackDefault.cpp instead? TextTrack.h has a
few of this variety.

[servolk, 2016/04/25 20:29:08]

Done.

+            // 5.2.4 Let audio label be a label specified in the initialization segment for this track or an empty string if no label info is present.
+            AtomicString label = trackInfo.label;
+            // 5.3.5 If audio label equals an empty string, then run the default track label algorithm with byteStreamTrackID set to audio byte stream track ID and
+            // type set to "audio" and assign the value returned by the algorithm to audio label.
+            if (label == "")
+                label = defaultTrackLabel("audio", byteStreamTrackId);
+            // 5.2.6 Let audio kinds be an array of kind strings specified in the initialization segment for this track or an empty array if no kind information is provided.
+            const auto& kind = trackInfo.kind;
+            // 5.2.8.2 Let new audio track be a new AudioTrack object.
+            AudioTrack* audioTrack = AudioTrack::create(byteStreamTrackId, kind, label, language, false);
+            SourceBufferTrackBaseSupplement::setSourceBuffer(*audioTrack, this);
+            assignedTrackIds[resultIdx++] = audioTrack->trackId();
+            // 5.2.8.7 If audioTracks.length equals 0, then run the following steps:
+            if (audioTracks().length() == 0) {
+                // 5.2.8.7.1 Set the enabled property on new audio track to true.
+                audioTrack->setEnabled(true);
+                // 5.2.8.7.2 Set active track flag to true.
+                m_activeTrack = true;
+            }
+            // 5.2.8.8 Add new audio track to the audioTracks attribute on this SourceBuffer object.
+            // 5.2.8.9 Queue a task to fire a trusted event named addtrack, that does not bubble and is not cancelable, and that uses the TrackEvent interface, at the AudioTrackList object referenced by the audioTracks attribute on this SourceBuffer object.
+            audioTracks().add(audioTrack);
+            // 5.2.8.10 Add new audio track to the audioTracks attribute on the HTMLMediaElement.
+            // 5.2.8.11 Queue a task to fire a trusted event named addtrack, that does not bubble and is not cancelable, and that uses the TrackEvent interface, at the AudioTrackList object referenced by the audioTracks attribute on the HTMLMediaElement.
+            m_source->mediaElement()->audioTracks().add(audioTrack);
+        }
+
+        // 5.3. For each video track in the initialization segment, run following steps:
+        for (const auto& trackInfo : newVideoTracks) {
+            // 5.3.1 Let video byte stream track ID be the Track ID for the current track being processed.
+            const auto& byteStreamTrackId = trackInfo.byteStreamTrackId;
+            // 5.3.2 Let video language be a BCP 47 language tag for the language specified in the initialization segment for this track or an empty string if no language info is present.
+            AtomicString language = trackInfo.language;
+            // 5.3.3 If video language equals an empty string or the 'und' BCP 47 value, then run the default track language algorithm with byteStreamTrackID set to
+            // video byte stream track ID and type set to "video" and assign the value returned by the algorithm to video language.
+            if (language == "" || language == "und")
+                language = defaultTrackLanguage("video", byteStreamTrackId);
+            // 5.3.4 Let video label be a label specified in the initialization segment for this track or an empty string if no label info is present.
+            AtomicString label = trackInfo.label;
+            // 5.3.5 If video label equals an empty string, then run the default track label algorithm with byteStreamTrackID set to video byte stream track ID and
+            // type set to "video" and assign the value returned by the algorithm to video label.
+            if (label == "")
+                label = defaultTrackLabel("video", byteStreamTrackId);
+            // 5.3.6 Let video kinds be an array of kind strings specified in the initialization segment for this track or an empty array if no kind information is provided.
+            const auto& kind = trackInfo.kind;
+            // 5.3.8.2 Let new video track be a new VideoTrack object.
+            VideoTrack* videoTrack = VideoTrack::create(byteStreamTrackId, kind, label, language, false);
+            SourceBufferTrackBaseSupplement::setSourceBuffer(*videoTrack, this);
+            assignedTrackIds[resultIdx++] = videoTrack->trackId();
+            // 5.3.8.7 If videoTracks.length equals 0, then run the following steps:
+            if (videoTracks().length() == 0) {
+                // 5.3.8.7.1 Set the selected property on new audio track to true.
+                videoTrack->setSelected(true);
+                // 5.3.8.7.2 Set active track flag to true.
+                m_activeTrack = true;
+            }
+            // 5.3.8.8 Add new video track to the videoTracks attribute on this SourceBuffer object.
+            // 5.3.8.9 Queue a task to fire a trusted event named addtrack, that does not bubble and is not cancelable, and that uses the TrackEvent interface, at the VideoTrackList object referenced by the videoTracks attribute on this SourceBuffer object.
+            videoTracks().add(videoTrack);
+            // 5.3.8.10 Add new video track to the videoTracks attribute on the HTMLMediaElement.
+            // 5.3.8.11 Queue a task to fire a trusted event named addtrack, that does not bubble and is not cancelable, and that uses the TrackEvent interface, at the VideoTrackList object referenced by the videoTracks attribute on the HTMLMediaElement.
+            m_source->mediaElement()->videoTracks().add(videoTrack);
+        }
+
+        // 5.4 TODO(servolk): Add text track processing here.
+
+        // 5.5 If active track flag equals true, then run the following steps:
         // activesourcebuffers.
-        m_source->setSourceBufferActive(this);
+        if (m_activeTrack) {
+            // 5.5.1 Add this SourceBuffer to activeSourceBuffers.
+            // 5.5.2 Queue a task to fire a simple event named addsourcebuffer at activeSourceBuffers
+            m_source->setSourceBufferActive(this);
+        }
 
-        // 6. Set first initialization segment received flag to true.
+        // 5.6. Set first initialization segment received flag to true.
         m_firstInitializationSegmentReceived = true;
     }
 
-    return result;
+    return assignedTrackIds;
 }
 
 bool SourceBuffer::hasPendingActivity() const
 {
     return m_source;
 }
 
 void SourceBuffer::suspend()
 {
     m_appendBufferAsyncPartRunner->suspend();
@@ skipping 367 matching lines @@
     visitor->trace(m_removeAsyncPartRunner);
     visitor->trace(m_appendStreamAsyncPartRunner);
     visitor->trace(m_stream);
     visitor->trace(m_audioTracks);
     visitor->trace(m_videoTracks);
     EventTargetWithInlineData::trace(visitor);
     ActiveDOMObject::trace(visitor);
 }
 
 } // namespace blink