Add gapless playback support for AAC playback.

media/filters/frame_processor.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/frame_processor.h"
 
 #include "base/stl_util.h"
 #include "media/base/buffers.h"
 #include "media/base/stream_parser_buffer.h"
 
@@ skipping 57 matching lines @@
   }
 
   // 2. - 4. Are handled by the WebMediaPlayer / Pipeline / Media Element.
 
   // Step 5:
   update_duration_cb_.Run(group_end_timestamp_);
 
   return true;
 }
 
-bool FrameProcessor::ProcessFrame(scoped_refptr<StreamParserBuffer> frame,
-                                  base::TimeDelta append_window_start,
-                                  base::TimeDelta append_window_end,
-                                  base::TimeDelta* timestamp_offset,
-                                  bool* new_media_segment) {
+bool FrameProcessor::ProcessFrame(
+    const scoped_refptr<StreamParserBuffer>& frame,
+    base::TimeDelta append_window_start,
+    base::TimeDelta append_window_end,
+    base::TimeDelta* timestamp_offset,
+    bool* new_media_segment) {
   // Implements the loop within step 1 of the coded frame processing algorithm
   // for a single input frame per April 1, 2014 MSE spec editor's draft:
   // https://dvcs.w3.org/hg/html-media/raw-file/d471a4412040/media-source/
   //     media-source.html#sourcebuffer-coded-frame-processing
 
   while (true) {
     // 1. Loop Top: Let presentation timestamp be a double precision floating
     //    point representation of the coded frame's presentation timestamp in
     //    seconds.
     // 2. Let decode timestamp be a double precision floating point
@@ skipping 61 matching lines @@
       //      true.
       SetAllTrackBuffersNeedRandomAccessPoint();
 
       // 4.4. Unset group start timestamp.
       group_start_timestamp_ = kNoTimestamp();
     }
 
     // 5. If timestampOffset is not 0, then run the following steps:
     if (*timestamp_offset != base::TimeDelta()) {
       // 5.1. Add timestampOffset to the presentation timestamp.
       // Note: |frame| PTS is only updated if it survives processing.

[wolenetz, 2014/05/30 19:58:45]

nit: s/survives/survives discontinuity/

[DaleCurtis, 2014/05/30 20:54:30]

Done.

       presentation_timestamp += *timestamp_offset;
 
       // 5.2. Add timestampOffset to the decode timestamp.
       // Frame DTS is only updated if it survives processing.

[wolenetz, 2014/05/30 19:58:45]

nit: ditto

[DaleCurtis, 2014/05/30 20:54:30]

Done.

       decode_timestamp += *timestamp_offset;
     }
 
     // 6. Let track buffer equal the track buffer that the coded frame will be
     //    added to.
 
     // Remap audio and video track types to their special singleton identifiers.
     StreamParser::TrackId track_id = kAudioTrackId;
     switch (frame->type()) {
       case DemuxerStream::AUDIO:
@@ skipping 65 matching lines @@
       DVLOG(2) << __FUNCTION__
                << ": frame PTS=" << presentation_timestamp.InSecondsF()
                << " or DTS=" << decode_timestamp.InSecondsF()
                << " negative after applying timestampOffset and handling any "
                << " discontinuity";
       return false;
     }
 
     // 9. Let frame end timestamp equal the sum of presentation timestamp and
     //    frame duration.
-    base::TimeDelta frame_end_timestamp = presentation_timestamp +
-        frame_duration;
+    const base::TimeDelta frame_end_timestamp =
+        presentation_timestamp + frame_duration;
 
     // 10.  If presentation timestamp is less than appendWindowStart, then set
     //      the need random access point flag to true, drop the coded frame, and
     //      jump to the top of the loop to start processing the next coded
     //      frame.
     // Note: We keep the result of partial discard of a buffer that overlaps
     //      |append_window_start| and does not end after |append_window_end|.
     // 11. If frame end timestamp is greater than appendWindowEnd, then set the
     //     need random access point flag to true, drop the coded frame, and jump
     //     to the top of the loop to start processing the next coded frame.
+    frame->set_timestamp(presentation_timestamp);
+    frame->SetDecodeTimestamp(decode_timestamp);
+    if (track_buffer->stream()->supports_partial_append_window_trimming() &&
+        HandlePartialAppendWindowTrimming(append_window_start,
+                                          append_window_end,
+                                          frame)) {
+      // |frame| has been partially trimmed or had preroll added.

[wolenetz, 2014/05/30 19:58:45]

I agree that the false path would set the flags for new media segment/random
access point needed, but in the case where no |audio_preroll_buffer_| was
previously held aside, and |frame| overlaps |append_window_start| and therefore
has been partially trimmed, we should still signal SBS of a new media segment
(if not in sequence mode). We shouldn't need to worry about setting
needs_random_access_point in this case since HandlePartialAppendWindowTrimming
DCHECKS the partially trimmed frame is a key frame. Net: Please add the
following here, perhaps with a clarifying comment.
if (!sequence_mode_)
  *new_media_segment = true;

[wolenetz, 2014/05/30 20:45:12]

From our chat, it's also possible that append window partial discard even in
sequence mode could introduce a gap such that SBS should be told a new media
segment is starting.
Please add: (here and also in LegacyFrameProcessor)
if (frame->timestamp() != presentation_timestamp) {
  *new_media_segment = true;
  track_buffer->set_needs_random_access_point(true);
}

[DaleCurtis, 2014/05/30 20:54:30]

Done.

+      decode_timestamp = frame->GetDecodeTimestamp();
+      presentation_timestamp = frame->timestamp();
+      frame_duration = frame->duration();
+
+      // The end timestamp of the frame should be unchanged.
+      DCHECK(frame_end_timestamp == presentation_timestamp + frame_duration);
+    }
+
     if (presentation_timestamp < append_window_start ||
         frame_end_timestamp > append_window_end) {
-      // See if a partial discard can be done around |append_window_start|.
-      // TODO(wolenetz): Refactor this into a base helper across legacy and
-      // new frame processors?
-      if (track_buffer->stream()->supports_partial_append_window_trimming() &&
-          presentation_timestamp < append_window_start &&
-          frame_end_timestamp > append_window_start &&
-          frame_end_timestamp <= append_window_end) {
-        DCHECK(frame->IsKeyframe());
-        DVLOG(1) << "Truncating buffer which overlaps append window start."
-                 << " presentation_timestamp "
-                 << presentation_timestamp.InSecondsF()
-                 << " append_window_start " << append_window_start.InSecondsF();
+      track_buffer->set_needs_random_access_point(true);
+      DVLOG(3) << "Dropping frame that is outside append window.";
 
-        // Adjust the timestamp of this frame forward to |append_window_start|,
-        // while decreasing the duration appropriately.
-        frame->set_discard_padding(std::make_pair(
-            append_window_start - presentation_timestamp, base::TimeDelta()));
-        presentation_timestamp = append_window_start;  // |frame| updated below.
-        decode_timestamp = append_window_start;  // |frame| updated below.
-        frame_duration = frame_end_timestamp - presentation_timestamp;
-        frame->set_duration(frame_duration);
+      if (!sequence_mode_) {

[wolenetz, 2014/05/30 20:45:12]

From our chat, this condition needs to be dropped. Even in sequence mode, a gap
could be introduced.

If MSE spec editors truly desire sequence mode append window filtering to filter
timestampOffset-adjusted-appended-buffers such that any buffers surviving the
filter get further relocated (and timestampOffset further adjusted) such that
they start without gap after previous group_end_timestamp, then I believe the
spec would need updating.

acolwell@, please comment on this.

[DaleCurtis, 2014/05/30 20:54:30]

I defer to you guys to sort this out outside of this patch.

[wolenetz, 2014/05/30 22:31:42]

SGTM. On further inspection, and per our offline chat, leaving this as-is
conforms to the intent of sequence mode. I retract my earlier comment here and
am working on a separate CL that clarifies new frame processor behavior w.r.t.
small append window discards.

[wolenetz, 2014/05/31 01:22:04]

I have a separate CL to test/clarify small append window discards:
https://codereview.chromium.org/310483005/

+        // This also triggers a discontinuity so we need to treat the next
+        // frames appended within the append window as if they were the
+        // beginning of a new segment.
+        *new_media_segment = true;
+      }
 
-        // TODO(dalecurtis): This could also be done with |append_window_end|,
-        // but is not necessary since splice frames covert the overlap there.
-      } else {
-        track_buffer->set_needs_random_access_point(true);
-        DVLOG(3) << "Dropping frame that is outside append window.";
-
-        if (!sequence_mode_) {
-          // This also triggers a discontinuity so we need to treat the next
-          // frames appended within the append window as if they were the
-          // beginning of a new segment.
-          *new_media_segment = true;
-        }
-
-        return true;
-      }
+      return true;
     }
 
     // 12. If the need random access point flag on track buffer equals true,
     //     then run the following steps:
     if (track_buffer->needs_random_access_point()) {
       // 12.1. If the coded frame is not a random access point, then drop the
       //       coded frame and jump to the top of the loop to start processing
       //       the next coded frame.
       if (!frame->IsKeyframe()) {
         DVLOG(3) << __FUNCTION__
                  << ": Dropping frame that is not a random access point";
         return true;
       }
 
       // 12.2. Set the need random access point flag on track buffer to false.
       track_buffer->set_needs_random_access_point(false);
     }
 
     // We now have a processed buffer to append to the track buffer's stream.
     // If it is the first in a new media segment or following a discontinuity,
     // notify all the track buffers' streams that a new segment is beginning.
     if (*new_media_segment) {
       *new_media_segment = false;
       NotifyNewMediaSegmentStarting(decode_timestamp);
     }
 
     DVLOG(3) << __FUNCTION__ << ": Sending processed frame to stream, "
              << "PTS=" << presentation_timestamp.InSecondsF()
              << ", DTS=" << decode_timestamp.InSecondsF();
-    frame->set_timestamp(presentation_timestamp);
-    frame->SetDecodeTimestamp(decode_timestamp);
 
     // Steps 13-18:
     // TODO(wolenetz): Collect and emit more than one buffer at a time, if
     // possible. Also refactor SourceBufferStream to conform to spec GC timing.
     // See http://crbug.com/371197.
     StreamParser::BufferQueue buffer_to_append;
     buffer_to_append.push_back(frame);
     track_buffer->stream()->Append(buffer_to_append);
 
     // 19. Set last decode timestamp for track buffer to decode timestamp.
     track_buffer->set_last_decode_timestamp(decode_timestamp);
 
     // 20. Set last frame duration for track buffer to frame duration.
     track_buffer->set_last_frame_duration(frame_duration);

[wolenetz, 2014/05/30 19:58:45]

fyi: A side effect of partial frame discard is that the updated (shorter)
track_buffer->last_frame_duration() will make subsequent buffer append that much
more sensitive to possible discontinuity, it it does not immediately abut
frame_end_timestamp. I doubt this will cause any big problem though.

 
     // 21. If highest presentation timestamp for track buffer is unset or frame
     //     end timestamp is greater than highest presentation timestamp, then
     //     set highest presentation timestamp for track buffer to frame end
     //     timestamp.
     track_buffer->SetHighestPresentationTimestampIfIncreased(
         frame_end_timestamp);
 
     // 22. If frame end timestamp is greater than group end timestamp, then set
     //     group end timestamp equal to frame end timestamp.
     DCHECK(group_end_timestamp_ >= base::TimeDelta());
     if (frame_end_timestamp > group_end_timestamp_)
       group_end_timestamp_ = frame_end_timestamp;
 
     return true;
   }
 
   NOTREACHED();
   return false;
 }
 
 void FrameProcessor::SetAllTrackBuffersNeedRandomAccessPoint() {
   for (TrackBufferMap::iterator itr = track_buffers_.begin();
        itr != track_buffers_.end(); ++itr) {
     itr->second->set_needs_random_access_point(true);
   }
 }
 
 }  // namespace media