MSE: Replace crossfade splicing overlap trimming.

media/filters/source_buffer_stream.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/source_buffer_stream.h"
 
 #include <algorithm>
 #include <map>
 #include <sstream>
+#include <string>
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/trace_event/trace_event.h"
 #include "media/base/audio_splicer.h"
 #include "media/base/timestamp_constants.h"
 #include "media/filters/source_buffer_platform.h"
 #include "media/filters/source_buffer_range.h"
 
 namespace media {
 
 namespace {
 
 // An arbitrarily-chosen number to estimate the duration of a buffer if none is
 // set and there's not enough information to get a better estimate.
 const int kDefaultBufferDurationInMs = 125;
 
-// Limit the number of MEDIA_LOG() logs for splice buffer generation warnings
-// and successes. Though these values are high enough to possibly exhaust the
-// media internals event cache (along with other events), these logs are
-// important for debugging splice generation.
-const int kMaxSpliceGenerationWarningLogs = 50;
-const int kMaxSpliceGenerationSuccessLogs = 20;
-
 // Limit the number of MEDIA_LOG() logs for track buffer time gaps.
 const int kMaxTrackBufferGapWarningLogs = 20;
 
 // Limit the number of MEDIA_LOG() logs for MSE GC algorithm warnings.
 const int kMaxGarbageCollectAlgorithmWarningLogs = 20;
 
+// Limit the number of MEDIA_LOG() logs for generated splice silence.
+const int kMaxSpliceGenerationLogs = 20;
+
 // Limit the number of MEDIA_LOG() logs for same DTS for non-keyframe followed
 // by keyframe. Prior to relaxing the "media segments must begin with a
 // keyframe" requirement, we issued decode error for this situation. That was
 // likely too strict, and now that the keyframe requirement is relaxed, we have
 // no knowledge of media segment boundaries here. Now, we log but don't trigger
 // decode error, since we allow these sequences which may cause extra decoder
 // work or other side-effects.
 const int kMaxStrangeSameTimestampsLogs = 20;
 
 // Helper method that returns true if |ranges| is sorted in increasing order,
@@ skipping 85 matching lines @@
       return SourceBufferRange::ALLOW_GAPS;
   }
 
   NOTREACHED();
   return SourceBufferRange::NO_GAPS_ALLOWED;
 }
 
 }  // namespace
 
 SourceBufferStream::SourceBufferStream(const AudioDecoderConfig& audio_config,
-                                       const scoped_refptr<MediaLog>& media_log,
-                                       bool splice_frames_enabled)
+                                       const scoped_refptr<MediaLog>& media_log)
     : media_log_(media_log),
       seek_buffer_timestamp_(kNoTimestamp),
       coded_frame_group_start_time_(kNoDecodeTimestamp()),
       range_for_next_append_(ranges_.end()),
       last_output_buffer_timestamp_(kNoDecodeTimestamp()),
       max_interbuffer_distance_(kNoTimestamp),
-      memory_limit_(kSourceBufferAudioMemoryLimit),
-      splice_frames_enabled_(splice_frames_enabled) {
+      memory_limit_(kSourceBufferAudioMemoryLimit) {
   DCHECK(audio_config.IsValidConfig());
   audio_configs_.push_back(audio_config);
 }
 
 SourceBufferStream::SourceBufferStream(const VideoDecoderConfig& video_config,
-                                       const scoped_refptr<MediaLog>& media_log,
-                                       bool splice_frames_enabled)
+                                       const scoped_refptr<MediaLog>& media_log)
     : media_log_(media_log),
       seek_buffer_timestamp_(kNoTimestamp),
       coded_frame_group_start_time_(kNoDecodeTimestamp()),
       range_for_next_append_(ranges_.end()),
       last_output_buffer_timestamp_(kNoDecodeTimestamp()),
       max_interbuffer_distance_(kNoTimestamp),
-      memory_limit_(kSourceBufferVideoMemoryLimit),
-      splice_frames_enabled_(splice_frames_enabled) {
+      memory_limit_(kSourceBufferVideoMemoryLimit) {
   DCHECK(video_config.IsValidConfig());
   video_configs_.push_back(video_config);
 }
 
 SourceBufferStream::SourceBufferStream(const TextTrackConfig& text_config,
-                                       const scoped_refptr<MediaLog>& media_log,
-                                       bool splice_frames_enabled)
+                                       const scoped_refptr<MediaLog>& media_log)
     : media_log_(media_log),
       text_track_config_(text_config),
       seek_buffer_timestamp_(kNoTimestamp),
       coded_frame_group_start_time_(kNoDecodeTimestamp()),
       range_for_next_append_(ranges_.end()),
       last_output_buffer_timestamp_(kNoDecodeTimestamp()),
       max_interbuffer_distance_(kNoTimestamp),
-      memory_limit_(kSourceBufferAudioMemoryLimit),
-      splice_frames_enabled_(splice_frames_enabled) {}
+      memory_limit_(kSourceBufferAudioMemoryLimit) {}
 
 SourceBufferStream::~SourceBufferStream() {
   while (!ranges_.empty()) {
     delete ranges_.front();
     ranges_.pop_front();
   }
 }
 
 void SourceBufferStream::OnStartOfCodedFrameGroup(
     DecodeTimestamp coded_frame_group_start_time) {
@@ skipping 62 matching lines @@
 
   // Save a snapshot of stream state before range modifications are made.
   DecodeTimestamp next_buffer_timestamp = GetNextBufferTimestamp();
   BufferQueue deleted_buffers;
 
   PrepareRangesForNextAppend(buffers, &deleted_buffers);
 
   // If there's a range for |buffers|, insert |buffers| accordingly. Otherwise,
   // create a new range with |buffers|.
   if (range_for_next_append_ != ranges_.end()) {
-    if (new_coded_frame_group_ && (!splice_frames_enabled_ ||
-                                   buffers.front()->splice_buffers().empty())) {
+    if (new_coded_frame_group_ &&
+        buffers.front()->leading_silence() == base::TimeDelta()) {
       // If the first append to this stream in a new coded frame group continues
       // a previous range, use the new group's start time instead of the first
       // new buffer's timestamp as the proof of adjacency to the existing range.
       // A large gap (larger than our normal buffer adjacency test) can occur in
       // a muxed set of streams (which share a common coded frame group start
       // time) with a significantly jagged start across the streams.
-      // Don't do this logic if there was a splice frame generated for the first
+      // Don't do this logic if splice silence was generated for the first
       // new buffer, since splices are guaranteed to be in the same range and
       // adjacent, and since the splice frame's timestamp can be less than
       // |coded_frame_group_start_time_| due to the splicing.
       (*range_for_next_append_)
           ->AppendBuffersToEnd(buffers, coded_frame_group_start_time_);
     } else {
       // Otherwise, use the first new buffer's timestamp as the proof of
       // adjacency.
       (*range_for_next_append_)
           ->AppendBuffersToEnd(buffers, kNoDecodeTimestamp());
@@ skipping 675 matching lines @@
     if (range_for_next_append_ != ranges_.begin()) {
       RangeList::iterator range_before_next = range_for_next_append_;
       --range_before_next;
       MergeWithAdjacentRangeIfNecessary(range_before_next);
     }
     MergeWithAdjacentRangeIfNecessary(range_for_next_append_);
   }
   return bytes_freed;
 }
 
+void SourceBufferStream::GenerateSpliceSilence(const BufferQueue& new_buffers) {
+  DCHECK(!new_buffers.empty());
+
+  // Find the overlapped range (if any).
+  const base::TimeDelta splice_timestamp = new_buffers.front()->timestamp();
+  const DecodeTimestamp splice_dts =
+      DecodeTimestamp::FromPresentationTime(splice_timestamp);
+  RangeList::iterator range_itr = FindExistingRangeFor(splice_dts);
+  if (range_itr == ranges_.end()) {
+    DVLOG(3) << __func__ << " Not generating splice. No range overlap at time "
+             << splice_timestamp.InMicroseconds();
+    return;
+  }
+
+  // Search for overlapped buffer needs exclusive end value. Choosing smallest
+  // possible value.
+  const DecodeTimestamp end_dts =
+      splice_dts + base::TimeDelta::FromInternalValue(1);
+
+  // Find if new buffer's start would overlap an existing buffer.
+  BufferQueue pre_splice_buffers;
+  if (!(*range_itr)
+           ->GetBuffersInRange(splice_dts, end_dts, &pre_splice_buffers)) {
+    // Bail if no overlapped buffers found.
+    DVLOG(3) << __func__ << " Not generating splice. No buffer overlap at time "
+             << splice_timestamp.InMicroseconds();
+    return;
+  }
+
+  // At most one buffer should exist containing the time of the newly appended
+  // buffer's start. GetBuffersInRange does not currently return buffers with
+  // zero duration.
+  DCHECK_EQ(pre_splice_buffers.size(), 1U)
+      << __func__ << " Found more than one overlapped buffer";
+
+  base::TimeDelta leading_silence = new_buffers.front()->timestamp() -
+                                    pre_splice_buffers.front()->timestamp();
+  DCHECK_GE(leading_silence, base::TimeDelta());
+
+  if (leading_silence == base::TimeDelta()) {
+    DVLOG(3) << __func__ << " Not generating splice for exact overlap at time "
+             << splice_timestamp.InMicroseconds();
+    return;
+  }
+
+  // Pad front of new buffers with leading silence and update front timestamp
+  // and duration. The overlapped buffer will be deleted in
+  // PrepareRangesForNewAppend.
+  new_buffers.front()->set_leading_silence(leading_silence);
+  new_buffers.front()->SetDecodeTimestamp(
+      new_buffers.front()->GetDecodeTimestamp() - leading_silence);
+  new_buffers.front()->set_timestamp(new_buffers.front()->timestamp() -
+                                     leading_silence);
+  new_buffers.front()->set_duration(new_buffers.front()->duration() +
+                                    leading_silence);
+
+  std::stringstream log_string;
+  log_string << "Generated splice for overlapping audio buffer at time:"
+             << splice_timestamp.InMicroseconds()
+             << "mu. Adding silence of length:"
+             << leading_silence.InMicroseconds()
+             << "mu to replace overlapped buffer at time:"
+             << pre_splice_buffers.front()->timestamp().InMicroseconds()
+             << "mu.";
+  LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_generation_logs_,
+                    kMaxSpliceGenerationLogs)
+      << log_string.str();
+  LOG(ERROR) << __func__ << log_string.str();
+}
+
 void SourceBufferStream::PrepareRangesForNextAppend(
     const BufferQueue& new_buffers, BufferQueue* deleted_buffers) {
   DCHECK(deleted_buffers);
 
-  // Handle splices between the existing buffers and the new buffers.  If a
-  // splice is generated the timestamp and duration of the first buffer in
-  // |new_buffers| will be modified.
-  if (splice_frames_enabled_)
-    GenerateSpliceFrame(new_buffers);
+  if (GetType() == kAudio)
+    GenerateSpliceSilence(new_buffers);
 
+  base::TimeDelta prev_duration = last_appended_buffer_duration_;
   DecodeTimestamp prev_timestamp = last_appended_buffer_timestamp_;
   DecodeTimestamp next_timestamp = new_buffers.front()->GetDecodeTimestamp();
 
+  // 1. Clean up the old buffers between the last appended buffer and the
+  //    beginning of |new_buffers|.
   if (prev_timestamp != kNoDecodeTimestamp() &&
       prev_timestamp != next_timestamp) {
-    // Clean up the old buffers between the last appended buffer and the
-    // beginning of |new_buffers|.
     RemoveInternal(prev_timestamp, next_timestamp, true, deleted_buffers);
   }
 
-  // Always make the start of the delete range exclusive for same timestamp
-  // across the last buffer in the previous append and the first buffer in the
-  // current append. Never be exclusive if a splice frame was generated because
-  // we don't generate splice frames for same timestamp situations.
-  DCHECK(new_buffers.front()->splice_timestamp() !=
-         new_buffers.front()->timestamp());
-  const bool exclude_start = new_buffers.front()->splice_buffers().empty() &&
-                             prev_timestamp == next_timestamp;
-
-  // Delete the buffers that |new_buffers| overlaps.
-  DecodeTimestamp start = new_buffers.front()->GetDecodeTimestamp();
+  // 2. Delete the buffers that |new_buffers| overlaps.
   if (new_coded_frame_group_) {
     // Extend the deletion range earlier to the coded frame group start time if
-    // this is the first append in a new coded frame group. Note that |start|
-    // could already be less than |coded_frame_group_start_time_| if a splice
-    // was generated.
+    // this is the first append in a new coded frame group. Note that
+    // |next_timestamp| could already be less than
+    // |coded_frame_group_start_time_| if splice silence was generated.
     DCHECK(coded_frame_group_start_time_ != kNoDecodeTimestamp());
-    start = std::min(coded_frame_group_start_time_, start);
+    next_timestamp = std::min(coded_frame_group_start_time_, next_timestamp);
   }
-  DecodeTimestamp end = new_buffers.back()->GetDecodeTimestamp();
-  base::TimeDelta duration = new_buffers.back()->duration();
+
+  // Exclude the start timestamp from removal if the previous append is a zero
+  // duration frame with the same timestamp as the current append. This avoids
+  // deleting zero-duration VP9 alt-ref frames while still deleting frames
+  // with non-zero duration that are now overlapped (potentially due to
+  // generated splice silence).
+  const bool exclude_start =
+      prev_timestamp == next_timestamp && prev_duration == base::TimeDelta();
 
   // Set end time for remove to include the duration of last buffer. If the
   // duration is estimated, use 1 microsecond instead to ensure frames are not
   // accidentally removed due to over-estimation.
+  DecodeTimestamp end = new_buffers.back()->GetDecodeTimestamp();
+  base::TimeDelta duration = new_buffers.back()->duration();
   if (duration != kNoTimestamp && duration > base::TimeDelta() &&
       !new_buffers.back()->is_duration_estimated()) {
     end += duration;
   } else {
     // TODO(chcunningham): Emit warning when 0ms durations are not expected.
     // http://crbug.com/312836
     end += base::TimeDelta::FromInternalValue(1);
   }
 
-  RemoveInternal(start, end, exclude_start, deleted_buffers);
+  // Finally do the deletion of overlap.
+  RemoveInternal(next_timestamp, end, exclude_start, deleted_buffers);
 }
 
 bool SourceBufferStream::AreAdjacentInSequence(
     DecodeTimestamp first_timestamp, DecodeTimestamp second_timestamp) const {
   return first_timestamp < second_timestamp &&
       second_timestamp <=
       first_timestamp + ComputeFudgeRoom(GetMaxInterbufferDistance());
 }
 
 void SourceBufferStream::PruneTrackBuffer(const DecodeTimestamp timestamp) {
@@ skipping 648 matching lines @@
     DVLOG(1) << __func__ << " deleting range_for_next_append_.";
     range_for_next_append_ = ranges_.end();
     last_appended_buffer_timestamp_ = kNoDecodeTimestamp();
     last_appended_buffer_is_keyframe_ = false;
   }
 
   delete **itr;
   *itr = ranges_.erase(*itr);
 }
 
-void SourceBufferStream::GenerateSpliceFrame(const BufferQueue& new_buffers) {
-  DCHECK(!new_buffers.empty());
-
-  // Splice frames are only supported for audio.
-  if (GetType() != kAudio)
-    return;
-
-  // Find the overlapped range (if any).
-  const base::TimeDelta splice_timestamp = new_buffers.front()->timestamp();
-  const DecodeTimestamp splice_dts =
-      DecodeTimestamp::FromPresentationTime(splice_timestamp);
-  RangeList::iterator range_itr = FindExistingRangeFor(splice_dts);
-  if (range_itr == ranges_.end())
-    return;
-
-  const DecodeTimestamp max_splice_end_dts =
-      splice_dts + base::TimeDelta::FromMilliseconds(
-          AudioSplicer::kCrossfadeDurationInMilliseconds);
-
-  // Find all buffers involved before the splice point.
-  BufferQueue pre_splice_buffers;
-  if (!(*range_itr)->GetBuffersInRange(
-          splice_dts, max_splice_end_dts, &pre_splice_buffers)) {
-    return;
-  }
-
-  // If there are gaps in the timeline, it's possible that we only find buffers
-  // after the splice point but within the splice range.  For simplicity, we do
-  // not generate splice frames in this case.
-  //
-  // We also do not want to generate splices if the first new buffer replaces an
-  // existing buffer exactly.
-  if (pre_splice_buffers.front()->timestamp() >= splice_timestamp) {
-    LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_generation_warning_logs_,
-                      kMaxSpliceGenerationWarningLogs)
-        << "Skipping splice frame generation: first new buffer at "
-        << splice_timestamp.InMicroseconds()
-        << "us begins at or before existing buffer at "
-        << pre_splice_buffers.front()->timestamp().InMicroseconds() << "us.";
-    DVLOG(1) << "Skipping splice: overlapped buffers begin at or after the "
-                "first new buffer.";
-    return;
-  }
-
-  // If any |pre_splice_buffers| are already splices or preroll, do not generate
-  // a splice.
-  for (size_t i = 0; i < pre_splice_buffers.size(); ++i) {
-    const BufferQueue& original_splice_buffers =
-        pre_splice_buffers[i]->splice_buffers();
-    if (!original_splice_buffers.empty()) {
-      LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_generation_warning_logs_,
-                        kMaxSpliceGenerationWarningLogs)
-          << "Skipping splice frame generation: overlapped buffers at "
-          << pre_splice_buffers[i]->timestamp().InMicroseconds()
-          << "us are in a previously buffered splice.";
-      DVLOG(1) << "Can't generate splice: overlapped buffers contain a "
-                  "pre-existing splice.";
-      return;
-    }
-
-    if (pre_splice_buffers[i]->preroll_buffer().get()) {
-      LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_generation_warning_logs_,
-                        kMaxSpliceGenerationWarningLogs)
-          << "Skipping splice frame generation: overlapped buffers at "
-          << pre_splice_buffers[i]->timestamp().InMicroseconds()
-          << "us contain preroll.";
-      DVLOG(1) << "Can't generate splice: overlapped buffers contain preroll.";
-      return;
-    }
-  }
-
-  // Don't generate splice frames which represent less than a millisecond (which
-  // is frequently the extent of timestamp resolution for poorly encoded media)
-  // or less than two samples (need at least two to crossfade).
-  const base::TimeDelta splice_duration =
-      pre_splice_buffers.back()->timestamp() +
-      pre_splice_buffers.back()->duration() - splice_timestamp;
-  const base::TimeDelta minimum_splice_duration = std::max(
-      base::TimeDelta::FromMilliseconds(1),
-      base::TimeDelta::FromSecondsD(
-          2.0 / audio_configs_[append_config_index_].samples_per_second()));
-  if (splice_duration < minimum_splice_duration) {
-    LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_generation_warning_logs_,
-                      kMaxSpliceGenerationWarningLogs)
-        << "Skipping splice frame generation: not enough samples for splicing "
-           "new buffer at "
-        << splice_timestamp.InMicroseconds() << "us. Have "
-        << splice_duration.InMicroseconds() << "us, but need "
-        << minimum_splice_duration.InMicroseconds() << "us.";
-    DVLOG(1) << "Can't generate splice: not enough samples for crossfade; have "
-             << splice_duration.InMicroseconds() << "us, but need "
-             << minimum_splice_duration.InMicroseconds() << "us.";
-    return;
-  }
-
-  DVLOG(1) << "Generating splice frame @ " << new_buffers.front()->timestamp()
-           << ", splice duration: " << splice_duration.InMicroseconds()
-           << " us";
-  LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_generation_success_logs_,
-                    kMaxSpliceGenerationSuccessLogs)
-      << "Generated splice of overlap duration "
-      << splice_duration.InMicroseconds() << "us into new buffer at "
-      << splice_timestamp.InMicroseconds() << "us.";
-  new_buffers.front()->ConvertToSpliceBuffer(pre_splice_buffers);
-}
-
 bool SourceBufferStream::SetPendingBuffer(
     scoped_refptr<StreamParserBuffer>* out_buffer) {
   DCHECK(out_buffer->get());
   DCHECK(!pending_buffer_.get());
 
   const bool have_splice_buffers = !(*out_buffer)->splice_buffers().empty();
   const bool have_preroll_buffer = !!(*out_buffer)->preroll_buffer().get();
 
   if (!have_splice_buffers && !have_preroll_buffer)
     return false;
 
   DCHECK_NE(have_splice_buffers, have_preroll_buffer);
   splice_buffers_index_ = 0;
   pending_buffer_.swap(*out_buffer);
   pending_buffers_complete_ = false;
   return true;
 }
 
 }  // namespace media