MSE: Relax the 'media segment must begin with keyframe' requirement

media/filters/frame_processor.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/frame_processor.h"
 
 #include <stdint.h>
 
 #include <cstdlib>
 
@@ skipping 179 matching lines @@
   // Step 8: Update the attribute to new mode.
   sequence_mode_ = sequence_mode;
 }
 
 bool FrameProcessor::ProcessFrames(
     const StreamParser::BufferQueue& audio_buffers,
     const StreamParser::BufferQueue& video_buffers,
     const StreamParser::TextBufferQueueMap& text_map,
     base::TimeDelta append_window_start,
     base::TimeDelta append_window_end,
-    bool* new_media_segment,
     base::TimeDelta* timestamp_offset) {
   StreamParser::BufferQueue frames;
   if (!MergeBufferQueues(audio_buffers, video_buffers, text_map, &frames)) {
     MEDIA_LOG(ERROR, media_log_) << "Parsed buffers not in DTS sequence";
     return false;
   }
 
   DCHECK(!frames.empty());
 
   // Implements the coded frame processing algorithm's outer loop for step 1.
   // Note that ProcessFrame() implements an inner loop for a single frame that
   // handles "jump to the Loop Top step to restart processing of the current
   // coded frame" per April 1, 2014 MSE spec editor's draft:
   // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/
   //     media-source.html#sourcebuffer-coded-frame-processing
   // 1. For each coded frame in the media segment run the following steps:
   for (StreamParser::BufferQueue::const_iterator frames_itr = frames.begin();
        frames_itr != frames.end(); ++frames_itr) {
     if (!ProcessFrame(*frames_itr, append_window_start, append_window_end,
-                      timestamp_offset, new_media_segment)) {
+                      timestamp_offset)) {
       FlushProcessedFrames();
       return false;
     }
   }
 
   if (!FlushProcessedFrames())
     return false;
 
   // 2. - 4. Are handled by the WebMediaPlayer / Pipeline / Media Element.
 
@@ skipping 56 matching lines @@
 void FrameProcessor::Reset() {
   DVLOG(2) << __FUNCTION__ << "()";
   for (TrackBufferMap::iterator itr = track_buffers_.begin();
        itr != track_buffers_.end(); ++itr) {
     itr->second->Reset();
   }
 
   if (sequence_mode_) {
     DCHECK(kNoTimestamp() != group_end_timestamp_);
     group_start_timestamp_ = group_end_timestamp_;
+    return;
   }
+
+  in_coded_frame_group_ = false;

[chcunningham, 2016/01/22 01:26:37]

Do you not reset this in sequence mode? (return a few lines up)

[wolenetz, 2016/01/22 01:50:03]

Correct. Sequence mode means everything appended is made to be in sequence, i.e.
in the same coded frame group. The "made to be" part includes dropping any
non-keyframes following a detected discontinuity, etc, and aligning buffer
timestamps to be in sequence, etc. Therefore, the result should be a continuous
sequence of coded frames within each track. Note that the
MseTrackBuffer::Reset() calls earlier in this FrameProcessor::Reset() still
allow ProcessFrame() to identify discontinuity (and also enforces that next
"passing" frame emitted by ProcessFrame() for each track *must* be a keyframe).

I suppose I could add a comment to the early return like "While in sequence
mode, maintain current state of in_coded_frame_group_ across this Reset()"

wdyt?

[wolenetz, 2016/01/22 18:59:31]

Done.

[chcunningham, 2016/01/22 19:26:02]

Acknowledged.

 }
 
 void FrameProcessor::OnPossibleAudioConfigUpdate(
     const AudioDecoderConfig& config) {
   DCHECK(config.IsValidConfig());
 
   // Always clear the preroll buffer when a config update is received.
   audio_preroll_buffer_ = NULL;
 
   if (config.Matches(current_audio_config_))
     return;
 
   current_audio_config_ = config;
   sample_duration_ = base::TimeDelta::FromSecondsD(
       1.0 / current_audio_config_.samples_per_second());
 }
 
 MseTrackBuffer* FrameProcessor::FindTrack(StreamParser::TrackId id) {
   TrackBufferMap::iterator itr = track_buffers_.find(id);
   if (itr == track_buffers_.end())
     return NULL;
 
   return itr->second;
 }
 
-void FrameProcessor::NotifyNewMediaSegmentStarting(
-    DecodeTimestamp segment_timestamp) {
-  DVLOG(2) << __FUNCTION__ << "(" << segment_timestamp.InSecondsF() << ")";
+void FrameProcessor::NotifyStartOfCodedFrameGroup(
+    DecodeTimestamp start_timestamp) {
+  DVLOG(2) << __FUNCTION__ << "(" << start_timestamp.InSecondsF() << ")";
 
   for (TrackBufferMap::iterator itr = track_buffers_.begin();
        itr != track_buffers_.end();
        ++itr) {
-    itr->second->stream()->OnNewMediaSegment(segment_timestamp);
+    itr->second->stream()->OnStartOfCodedFrameGroup(start_timestamp);
   }
 }
 
 bool FrameProcessor::FlushProcessedFrames() {
   DVLOG(2) << __FUNCTION__ << "()";
 
   bool result = true;
   for (TrackBufferMap::iterator itr = track_buffers_.begin();
        itr != track_buffers_.end();
        ++itr) {
@@ skipping 100 matching lines @@
     processed_buffer = true;
   }
 
   return processed_buffer;
 }
 
 bool FrameProcessor::ProcessFrame(
     const scoped_refptr<StreamParserBuffer>& frame,
     base::TimeDelta append_window_start,
     base::TimeDelta append_window_end,
-    base::TimeDelta* timestamp_offset,
-    bool* new_media_segment) {
+    base::TimeDelta* timestamp_offset) {
   // Implements the loop within step 1 of the coded frame processing algorithm
   // for a single input frame per April 1, 2014 MSE spec editor's draft:
   // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/
   //     media-source.html#sourcebuffer-coded-frame-processing
 
   while (true) {
     // 1. Loop Top: Let presentation timestamp be a double precision floating
     //    point representation of the coded frame's presentation timestamp in
     //    seconds.
     // 2. Let decode timestamp be a double precision floating point
@@ skipping 117 matching lines @@
           << ", and parser track id " << frame->track_id();
       return false;
     }
 
     // 7. If last decode timestamp for track buffer is set and decode timestamp
     //    is less than last decode timestamp
     //    OR
     //    If last decode timestamp for track buffer is set and the difference
     //    between decode timestamp and last decode timestamp is greater than 2
     //    times last frame duration:
-    DecodeTimestamp last_decode_timestamp =
+    DecodeTimestamp track_last_decode_timestamp =
         track_buffer->last_decode_timestamp();
-    if (last_decode_timestamp != kNoDecodeTimestamp()) {
-      base::TimeDelta dts_delta = decode_timestamp - last_decode_timestamp;
-      if (dts_delta < base::TimeDelta() ||
-          dts_delta > 2 * track_buffer->last_frame_duration()) {
+    if (track_last_decode_timestamp != kNoDecodeTimestamp()) {
+      base::TimeDelta track_dts_delta =
+          decode_timestamp - track_last_decode_timestamp;
+      if (track_dts_delta < base::TimeDelta() ||
+          track_dts_delta > 2 * track_buffer->last_frame_duration()) {
+        DCHECK(in_coded_frame_group_);
         // 7.1. If mode equals "segments": Set group end timestamp to
         //      presentation timestamp.
         //      If mode equals "sequence": Set group start timestamp equal to
         //      the group end timestamp.
         if (!sequence_mode_) {
           group_end_timestamp_ = presentation_timestamp;
           // This triggers a discontinuity so we need to treat the next frames
           // appended within the append window as if they were the beginning of
-          // a new segment.
-          *new_media_segment = true;
+          // a new coded frame group.
+          in_coded_frame_group_ = false;
         } else {
           DVLOG(3) << __FUNCTION__ << " : Sequence mode discontinuity, GETS: "
                    << group_end_timestamp_.InSecondsF();
           DCHECK(kNoTimestamp() != group_end_timestamp_);
           group_start_timestamp_ = group_end_timestamp_;
         }
 
         // 7.2. - 7.5.:
         Reset();
 
@@ skipping 69 matching lines @@
         DVLOG(3) << __FUNCTION__
                  << ": Dropping frame that is not a random access point";
         return true;
       }
 
       // 12.2. Set the need random access point flag on track buffer to false.
       track_buffer->set_needs_random_access_point(false);
     }
 
     // We now have a processed buffer to append to the track buffer's stream.
-    // If it is the first in a new media segment or following a discontinuity,
-    // notify all the track buffers' streams that a new segment is beginning.
-    if (*new_media_segment) {
-      // First, complete the append to track buffer streams of previous media
-      // segment's frames, if any.
+    // If it is the first in a new coded frame group (such as following a
+    // discontinuity), notify all the track buffers' streams that a coded frame
+    // group is starting.
+    if (!in_coded_frame_group_) {
+      // First, complete the append to track buffer streams of the previous
+      // coded frame group's frames, if any.
       if (!FlushProcessedFrames())
         return false;
 
-      *new_media_segment = false;
-
       // TODO(acolwell/wolenetz): This should be changed to a presentation
       // timestamp. See http://crbug.com/402502
-      NotifyNewMediaSegmentStarting(decode_timestamp);
+      NotifyStartOfCodedFrameGroup(decode_timestamp);
+      in_coded_frame_group_ = true;
     }
 
     DVLOG(3) << __FUNCTION__ << ": Sending processed frame to stream, "
              << "PTS=" << presentation_timestamp.InSecondsF()
              << ", DTS=" << decode_timestamp.InSecondsF();
 
     // Steps 13-18: Note, we optimize by appending groups of contiguous
     // processed frames for each track buffer at end of ProcessFrames() or prior
-    // to NotifyNewMediaSegmentStarting().
-    // TODO(wolenetz): Refactor SourceBufferStream to conform to spec GC timing.
-    // See http://crbug.com/371197.
+    // to NotifyStartOfCodedFrameGroup().
     track_buffer->EnqueueProcessedFrame(frame);
 
     // 19. Set last decode timestamp for track buffer to decode timestamp.
     track_buffer->set_last_decode_timestamp(decode_timestamp);
 
     // 20. Set last frame duration for track buffer to frame duration.
     track_buffer->set_last_frame_duration(frame_duration);
 
     // 21. If highest presentation timestamp for track buffer is unset or frame
     //     end timestamp is greater than highest presentation timestamp, then
     //     set highest presentation timestamp for track buffer to frame end
     //     timestamp.
     track_buffer->SetHighestPresentationTimestampIfIncreased(
         frame_end_timestamp);
 
     // 22. If frame end timestamp is greater than group end timestamp, then set
     //     group end timestamp equal to frame end timestamp.
     if (frame_end_timestamp > group_end_timestamp_)
       group_end_timestamp_ = frame_end_timestamp;
     DCHECK(group_end_timestamp_ >= base::TimeDelta());
 
     return true;
   }
 
   NOTREACHED();
   return false;
 }
 
 }  // namespace media