Fix MSE GC, make it less aggressive, more spec-compliant

media/filters/chunk_demuxer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/chunk_demuxer.h"
 
 #include <algorithm>
 #include <limits>
 #include <list>
 
@@ skipping 113 matching lines @@
 
   // Aborts the current append sequence and resets the parser.
   void Abort(TimeDelta append_window_start,
              TimeDelta append_window_end,
              TimeDelta* timestamp_offset);
 
   // Calls Remove(|start|, |end|, |duration|) on all
   // ChunkDemuxerStreams managed by this object.
   void Remove(TimeDelta start, TimeDelta end, TimeDelta duration);
 
+  // If the buffer is full, attempts to try to free up space, as specified in
+  // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec.
+  // Returns false iff buffer is still full after running eviction.
+  // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction
+  bool EvictCodedFrames(DecodeTimestamp media_time, size_t newDataSize);
+
   // Returns true if currently parsing a media segment, or false otherwise.
   bool parsing_media_segment() const { return parsing_media_segment_; }
 
   // Sets |frame_processor_|'s sequence mode to |sequence_mode|.
   void SetSequenceMode(bool sequence_mode);
 
   // Signals the coded frame processor to update its group start timestamp to be
   // |timestamp_offset| if it is in sequence append mode.
   void SetGroupStartTimestampIfInSequenceMode(base::TimeDelta timestamp_offset);
 
@@ skipping 44 matching lines @@
   // appending the processed frames to associated demuxer streams for each
   // frame's track.
   // Returns true on a successful call. Returns false if an error occurred while
   // processing the buffers.
   bool OnNewBuffers(const StreamParser::BufferQueue& audio_buffers,
                     const StreamParser::BufferQueue& video_buffers,
                     const StreamParser::TextBufferQueueMap& text_map);
 
   void OnSourceInitDone(const StreamParser::InitParameters& params);
 
+  // EstimateVideoDataSize uses some heuristics to estimate the size of the
+  // video size in the chunk of muxed audio/video data without parsing it.
+  // This is used by EvictCodedFrames algorithm, which happens before Append
+  // (and therefore before parsing is performed) to prepare space for new data.
+  size_t EstimateVideoDataSize(size_t muxed_data_chunk_size) const;
+
   CreateDemuxerStreamCB create_demuxer_stream_cb_;
   NewTextTrackCB new_text_track_cb_;
 
   // During Append(), if OnNewBuffers() coded frame processing updates the
   // timestamp offset then |*timestamp_offset_during_append_| is also updated
   // so Append()'s caller can know the new offset. This pointer is only non-NULL
   // during the lifetime of an Append() call.
   TimeDelta* timestamp_offset_during_append_;
 
   // During Append(), coded frame processing triggered by OnNewBuffers()
@@ skipping 154 matching lines @@
 
   if (video_)
     video_->Remove(start, end, duration);
 
   for (TextStreamMap::iterator itr = text_stream_map_.begin();
        itr != text_stream_map_.end(); ++itr) {
     itr->second->Remove(start, end, duration);
   }
 }
 
+size_t SourceState::EstimateVideoDataSize(size_t muxed_data_chunk_size) const {
+  DCHECK(audio_);
+  DCHECK(video_);
+
+  size_t bufferedVideoSize = video_->GetBufferedSize();
+  size_t bufferedAudioSize = audio_->GetBufferedSize();
+  if (bufferedVideoSize == 0 || bufferedAudioSize == 0) {
+    // At this point either audio or video buffer is empty, which means buffer
+    // levels are probably low anyway and we should have enough space in the
+    // buffers for appending new data, so just take a very rough guess.
+    return muxed_data_chunk_size / 2;
+  }
+
+  // We need to estimate how much audio and video data is going to be in the
+  // newly appended data chunk to make space for the new data. And we need to do
+  // that without parsing the data (which will happen later, in the Append
+  // phase). So for now we can only rely on some heuristic here. Let's assume
+  // that the proportion of the audio/video in the new data chunk is the same as
+  // the current ratio of buffered audio/video.
+  // Longer term this should go away once we further change the MSE GC algorithm
+  // to work across all streams of a SourceBuffer (see crbug.com/520704).
+  DCHECK(bufferedVideoSize + bufferedAudioSize > 0);

[wolenetz, 2015/08/21 18:15:57]

Hmm. size_t can't be negative. So this is only checking that there is something
buffered, right?

[servolk, 2015/08/21 18:19:14]

Right

[servolk, 2015/08/21 19:06:45]

(this is just a sanity check, since we are going to divide by that number in the
next line)

[wolenetz, 2015/08/21 19:39:37]

Acknowledged.
Let's please strengthen sanity here by ensuring the addition doesn't overflow.
something like DCHECK(max_size_t - bufferedVideoSize > bufferedAudioSize)

[servolk, 2015/08/21 20:16:54]

Done.

+  size_t estimatedVideoData = muxed_data_chunk_size * bufferedVideoSize /

[wolenetz, 2015/08/21 18:15:56]

not lgtm:
Working with integers, this may not do what you want:

100MB existing, 51MB video, 49MB audio. 10MB new. I assume you'd want 4.9MB of
estimated audio headroom and 5.1MB of estimated video headroom (the latter being
the output of this helper method).
This method would instead yield:
estimated video data size == 10MB * (51MB / 100MB) == 10MB * (0) == 0

[servolk, 2015/08/21 18:19:14]

I believe it would be calculated as (10 * 51) / 100 = 5. I can either add extra
parentheses around multiplication or use floating-point calculations. Which one
do you like better?

[servolk, 2015/08/21 19:06:45]

Since I had to upload a new patchset with the updated unit tests anyway, I've
added parens here to make this clearer.

[wolenetz, 2015/08/21 19:39:37]

Acknowledged.

+      (bufferedVideoSize + bufferedAudioSize);
+  return estimatedVideoData;
+}
+
+bool SourceState::EvictCodedFrames(DecodeTimestamp media_time,
+                                   size_t newDataSize) {
+  bool success = true;
+
+  DVLOG(3) << __FUNCTION__ << " media_time=" << media_time.InSecondsF()
+           << " newDataSize=" << newDataSize
+           << " bufferedVideoSize=" << (video_ ? video_->GetBufferedSize() : 0)
+           << " bufferedAudioSize=" << (audio_ ? audio_->GetBufferedSize() : 0);
+
+  size_t newAudioSize = 0;
+  size_t newVideoSize = 0;
+  if (audio_ && video_) {
+    newVideoSize = EstimateVideoDataSize(newDataSize);
+    newAudioSize = newDataSize - newVideoSize;
+  } else if (video_) {
+    newVideoSize = newDataSize;
+  } else if (audio_) {
+    newAudioSize = newDataSize;
+  }
+
+  DVLOG(3) << __FUNCTION__ << " estimated audio/video sizes: "
+           << " newVideoSize=" << newVideoSize
+           << " newAudioSize=" << newAudioSize;
+
+  if (audio_)
+    success = audio_->EvictCodedFrames(media_time, newAudioSize) && success;
+
+  if (video_)
+    success = video_->EvictCodedFrames(media_time, newVideoSize) && success;
+
+  for (TextStreamMap::iterator itr = text_stream_map_.begin();
+       itr != text_stream_map_.end(); ++itr) {
+    success = itr->second->EvictCodedFrames(media_time, 0) && success;
+  }
+
+  DVLOG(3) << __FUNCTION__ << " result=" << success
+           << " bufferedVideoSize=" << (video_ ? video_->GetBufferedSize() : 0)
+           << " bufferedAudioSize=" << (audio_ ? audio_->GetBufferedSize() : 0);
+
+  return success;
+}
+
 Ranges<TimeDelta> SourceState::GetBufferedRanges(TimeDelta duration,
                                                  bool ended) const {
   // TODO(acolwell): When we start allowing disabled tracks we'll need to update
   // this code to only add ranges from active tracks.
   RangesList ranges_list;
   if (audio_)
     ranges_list.push_back(audio_->GetBufferedRanges(duration));
 
   if (video_)
     ranges_list.push_back(video_->GetBufferedRanges(duration));
@@ skipping 490 matching lines @@
 
   return true;
 }
 
 void ChunkDemuxerStream::Remove(TimeDelta start, TimeDelta end,
                                 TimeDelta duration) {
   base::AutoLock auto_lock(lock_);
   stream_->Remove(start, end, duration);
 }
 
+bool ChunkDemuxerStream::EvictCodedFrames(DecodeTimestamp media_time,
+                                          size_t newDataSize) {
+  base::AutoLock auto_lock(lock_);
+  return stream_->GarbageCollectIfNeeded(media_time, newDataSize);
+}
+
 void ChunkDemuxerStream::OnSetDuration(TimeDelta duration) {
   base::AutoLock auto_lock(lock_);
   stream_->OnSetDuration(duration);
 }
 
 Ranges<TimeDelta> ChunkDemuxerStream::GetBufferedRanges(
     TimeDelta duration) const {
   base::AutoLock auto_lock(lock_);
 
   if (type_ == TEXT) {
@@ skipping 16 matching lines @@
   // range.
   Ranges<TimeDelta> valid_time_range;
   valid_time_range.Add(range.start(0), duration);
   return range.IntersectionWith(valid_time_range);
 }
 
 TimeDelta ChunkDemuxerStream::GetBufferedDuration() const {
   return stream_->GetBufferedDuration();
 }
 
+size_t ChunkDemuxerStream::GetBufferedSize() const {
+  return stream_->GetBufferedSize();
+}
+
 void ChunkDemuxerStream::OnNewMediaSegment(DecodeTimestamp start_timestamp) {
   DVLOG(2) << "ChunkDemuxerStream::OnNewMediaSegment("
            << start_timestamp.InSecondsF() << ")";
   base::AutoLock auto_lock(lock_);
   stream_->OnNewMediaSegment(start_timestamp);
 }
 
 bool ChunkDemuxerStream::UpdateAudioConfig(
     const AudioDecoderConfig& config,
     const scoped_refptr<MediaLog>& media_log) {
@@ skipping 386 matching lines @@
 Ranges<TimeDelta> ChunkDemuxer::GetBufferedRanges(const std::string& id) const {
   base::AutoLock auto_lock(lock_);
   DCHECK(!id.empty());
 
   SourceStateMap::const_iterator itr = source_state_map_.find(id);
 
   DCHECK(itr != source_state_map_.end());
   return itr->second->GetBufferedRanges(duration_, state_ == ENDED);
 }
 
+bool ChunkDemuxer::EvictCodedFrames(const std::string& id,
+                                    base::TimeDelta currentMediaTime,
+                                    size_t newDataSize) {
+  DVLOG(1) << __FUNCTION__ << "(" << id << ")"
+           << " media_time=" << currentMediaTime.InSecondsF()
+           << " newDataSize=" << newDataSize;
+  base::AutoLock auto_lock(lock_);
+
+  // Note: The direct conversion from PTS to DTS is safe here, since we don't
+  // need to know currentTime precisely for GC. GC only needs to know which GOP
+  // currentTime points to.
+  DecodeTimestamp media_time_dts =
+      DecodeTimestamp::FromPresentationTime(currentMediaTime);
+
+  DCHECK(!id.empty());
+  SourceStateMap::const_iterator itr = source_state_map_.find(id);
+  if (itr == source_state_map_.end()) {
+    LOG(WARNING) << __FUNCTION__ << " stream " << id << " not found";
+    return false;
+  }
+  return itr->second->EvictCodedFrames(media_time_dts, newDataSize);
+}
+
 void ChunkDemuxer::AppendData(
     const std::string& id,
     const uint8* data,
     size_t length,
     TimeDelta append_window_start,
     TimeDelta append_window_end,
     TimeDelta* timestamp_offset,
     const InitSegmentReceivedCB& init_segment_received_cb) {
   DVLOG(1) << "AppendData(" << id << ", " << length << ")";
 
@@ skipping 500 matching lines @@
 }
 
 void ChunkDemuxer::ShutdownAllStreams() {
   for (SourceStateMap::iterator itr = source_state_map_.begin();
        itr != source_state_map_.end(); ++itr) {
     itr->second->Shutdown();
   }
 }
 
 }  // namespace media