Reland: MSE: Relax the 'media segment must begin with keyframe' requirement

media/filters/chunk_demuxer_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/chunk_demuxer.h"
 
 #include <stddef.h>
 #include <stdint.h>
 #include <algorithm>
 #include <utility>
@@ skipping 667 matching lines @@
   void AppendMuxedCluster(const MuxedStreamInfo& msi_1,
                           const MuxedStreamInfo& msi_2,
                           const MuxedStreamInfo& msi_3) {
     std::vector<MuxedStreamInfo> msi(3);
     msi[0] = msi_1;
     msi[1] = msi_2;
     msi[2] = msi_3;
     AppendMuxedCluster(msi);
   }
 
-  void AppendMuxedCluster(const std::vector<MuxedStreamInfo> msi) {
+  scoped_ptr<Cluster> GenerateMuxedCluster(
+      const std::vector<MuxedStreamInfo> msi) {
     std::priority_queue<BlockInfo> block_queue;
     for (size_t i = 0; i < msi.size(); ++i) {
       std::vector<BlockInfo> track_blocks;
       ParseBlockDescriptions(msi[i].track_number, msi[i].block_descriptions,
                              &track_blocks);
 
       for (size_t j = 0; j < track_blocks.size(); ++j) {
         block_queue.push(track_blocks[j]);
       }
 
       if (msi[i].last_blocks_estimated_duration != -1) {
         EXPECT_MEDIA_LOG(WebMSimpleBlockDurationEstimated(
             msi[i].last_blocks_estimated_duration));
       }
     }
+    return GenerateCluster(block_queue, false);
+  }
 
-    AppendCluster(kSourceId, GenerateCluster(block_queue, false));
+  void AppendMuxedCluster(const std::vector<MuxedStreamInfo> msi) {
+    AppendCluster(kSourceId, GenerateMuxedCluster(msi));
   }
 
   void AppendData(const std::string& source_id,
                   const uint8_t* data,
                   size_t length) {
     EXPECT_CALL(host_, OnBufferedTimeRangesChanged(_)).Times(AnyNumber());
 
     demuxer_->AppendData(source_id, data, length,
                          append_window_start_for_next_append_,
                          append_window_end_for_next_append_,
@@ skipping 179 matching lines @@
 
   // Initializes the demuxer with data from 2 files with different
   // decoder configurations. This is used to test the decoder config change
   // logic.
   //
   // bear-320x240.webm VideoDecoderConfig returns 320x240 for its natural_size()
   // bear-640x360.webm VideoDecoderConfig returns 640x360 for its natural_size()
   // The resulting video stream returns data from each file for the following
   // time ranges.
   // bear-320x240.webm : [0-501)       [801-2736)
-  // bear-640x360.webm :       [527-793)
+  // bear-640x360.webm :       [527-760)

[wolenetz, 2016/02/12 01:23:47]

This is intentional. As of patch set 4-5, the removal range for a new coded
frame group append extends earlier in time potentially than the first new frame
for the track in the group (it can go back to the coded frame group start time.)
Also, the test media includes audio discontinuities at every webm cluster
boundary, triggering frequent new coded frame groups (and removal logic that now
removes a little more).
ChunkDemuxerTest.ConfigChange_Video is also adjusted to match this.

[wolenetz, 2016/02/24 00:34:46]

Note this earlier comment was meant to help explain the related
ConfigChange_Video test change.

[chcunningham, 2016/02/24 19:06:26]

Acknowledged.

   //
   // bear-320x240.webm AudioDecoderConfig returns 3863 for its extra_data size.
   // bear-640x360.webm AudioDecoderConfig returns 3935 for its extra_data size.
   // The resulting audio stream returns data from each file for the following
   // time ranges.
   // bear-320x240.webm : [0-524)       [779-2736)
   // bear-640x360.webm :       [527-759)
   bool InitDemuxerWithConfigChangeData() {
     scoped_refptr<DecoderBuffer> bear1 = ReadTestDataFile("bear-320x240.webm");
     scoped_refptr<DecoderBuffer> bear2 = ReadTestDataFile("bear-640x360.webm");
@@ skipping 2278 matching lines @@
   const VideoDecoderConfig& video_config_2 = video->video_decoder_config();
   ASSERT_TRUE(video_config_2.IsValidConfig());
   EXPECT_EQ(video_config_2.natural_size().width(), 640);
   EXPECT_EQ(video_config_2.natural_size().height(), 360);
 
   ExpectRead(DemuxerStream::VIDEO, 527);
 
   // Read until the next config change.
   ReadUntilNotOkOrEndOfStream(DemuxerStream::VIDEO, &status, &last_timestamp);
   ASSERT_EQ(status, DemuxerStream::kConfigChanged);
-  EXPECT_EQ(last_timestamp.InMilliseconds(), 793);
+  EXPECT_EQ(last_timestamp.InMilliseconds(), 760);
 
   // Get the new config and verify that it matches the first one.
   ASSERT_TRUE(video_config_1.Matches(video->video_decoder_config()));
 
   ExpectRead(DemuxerStream::VIDEO, 801);
 
   // Read until the end of the stream just to make sure there aren't any other
   // config changes.
   ReadUntilNotOkOrEndOfStream(DemuxerStream::VIDEO, &status, &last_timestamp);
   ASSERT_EQ(status, DemuxerStream::kOk);
@@ skipping 1179 matching lines @@
   EXPECT_MEDIA_LOG(SegmentMissingFrames("video"));
   AppendSingleStreamCluster(kSourceId, kAudioTrackNum, 0, 10);
 }
 
 TEST_F(ChunkDemuxerTest, SegmentMissingAudioVideoFrames) {
   ASSERT_TRUE(InitDemuxer(HAS_AUDIO | HAS_VIDEO));
   EXPECT_MEDIA_LOG(SegmentMissingFrames("audio or video"));
   AppendCluster(GenerateEmptyCluster(0));
 }
 
+TEST_F(ChunkDemuxerTest, RelaxedKeyframe_FirstSegmentMissingKeyframe) {
+  // Append V:[n n n][n n K]
+  // Expect V:           [K]
+  ASSERT_TRUE(InitDemuxer(HAS_VIDEO));
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  EXPECT_MEDIA_LOG(WebMSimpleBlockDurationEstimated(10)).Times(2);
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "0 10 20");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "30 40 50K");
+  CheckExpectedRanges("{ [50,60) }");
+  CheckExpectedBuffers(video_stream, "50K");
+}
+
+TEST_F(ChunkDemuxerTest, RelaxedKeyframe_SecondSegmentMissingKeyframe) {
+  // Append V:[K n n][n n n]
+  // Expect V:[K n n][n n n]
+  ASSERT_TRUE(InitDemuxer(HAS_VIDEO));
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  EXPECT_MEDIA_LOG(WebMSimpleBlockDurationEstimated(10)).Times(2);
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "0K 10 20");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "30 40 50");
+  CheckExpectedRanges("{ [0,60) }");
+  CheckExpectedBuffers(video_stream, "0K 10 20 30 40 50");
+}
+
+TEST_F(ChunkDemuxerTest, RelaxedKeyframe_RemoveInterruptsCodedFrameGroup_1) {
+  // Append V:[K n n]
+  // Remove    *****
+  // Append V:       [n n n][n K n]
+  // Expect:                  [K n]
+  ASSERT_TRUE(InitDemuxer(HAS_VIDEO));
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  EXPECT_MEDIA_LOG(WebMSimpleBlockDurationEstimated(10)).Times(3);
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "0K 10 20");
+  demuxer_->Remove(kSourceId, base::TimeDelta(),
+                   base::TimeDelta::FromMilliseconds(30));
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "30 40 50");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "60 70K 80");
+  CheckExpectedRanges("{ [70,90) }");
+  CheckExpectedBuffers(video_stream, "70K 80");
+}
+
+TEST_F(ChunkDemuxerTest, RelaxedKeyframe_RemoveInterruptsCodedFrameGroup_2) {
+  // Append V:[K n n][n n n][n K n]
+  // Remove    *
+  // Expect:                  [K n]
+  ASSERT_TRUE(InitDemuxer(HAS_VIDEO));
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  EXPECT_MEDIA_LOG(WebMSimpleBlockDurationEstimated(10)).Times(3);
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "0K 10 20");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "30 40 50");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "60 70K 80");
+  demuxer_->Remove(kSourceId, base::TimeDelta(),
+                   base::TimeDelta::FromMilliseconds(10));
+  CheckExpectedRanges("{ [70,90) }");
+  CheckExpectedBuffers(video_stream, "70K 80");
+}
+
+TEST_F(ChunkDemuxerTest, RelaxedKeyframe_RemoveInterruptsCodedFrameGroup_3) {
+  // Append V:[K n n][n n n][n K n]
+  // Remove               *
+  // Expect:  [K n n..n n]    [K n]
+  ASSERT_TRUE(InitDemuxer(HAS_VIDEO));
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  EXPECT_MEDIA_LOG(WebMSimpleBlockDurationEstimated(10)).Times(3);
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "0K 10 20");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "30 40 50");
+  AppendSingleStreamCluster(kSourceId, kVideoTrackNum, "60 70K 80");
+  demuxer_->Remove(kSourceId, base::TimeDelta::FromMilliseconds(50),
+                   base::TimeDelta::FromMilliseconds(60));
+  CheckExpectedRanges("{ [0,50) [70,90) }");
+  CheckExpectedBuffers(video_stream, "0K 10 20 30 40");
+  Seek(base::TimeDelta::FromMilliseconds(70));
+  CheckExpectedBuffers(video_stream, "70K 80");
+}
+
+TEST_F(ChunkDemuxerTest,
+       RelaxedKeyframe_RemoveInterruptsMuxedCodedFrameGroup_1) {
+  // Append muxed:
+  //        A:[K K K]
+  //        V:[K n n]
+  // Remove    *****
+  // Append muxed:
+  //        A:       [K K K][K K K]
+  //        V:       [n n n][n K n]
+  // Expect:
+  //        A:       [K K K][K K K]
+  //        V                 [K n]
+  ASSERT_TRUE(InitDemuxer(HAS_AUDIO | HAS_VIDEO));
+  DemuxerStream* audio_stream = demuxer_->GetStream(DemuxerStream::AUDIO);
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  AppendMuxedCluster(MuxedStreamInfo(kAudioTrackNum, "0K 10K 20D10K"),
+                     MuxedStreamInfo(kVideoTrackNum, "0K 10 20", 10));
+  demuxer_->Remove(kSourceId, base::TimeDelta(),
+                   base::TimeDelta::FromMilliseconds(30));
+  AppendMuxedCluster(MuxedStreamInfo(kAudioTrackNum, "30K 40K 50D10K"),
+                     MuxedStreamInfo(kVideoTrackNum, "30 40 50", 10));
+  AppendMuxedCluster(MuxedStreamInfo(kAudioTrackNum, "60K 70K 80D10K"),
+                     MuxedStreamInfo(kVideoTrackNum, "60 70K 80", 10));
+  CheckExpectedRanges(DemuxerStream::AUDIO, "{ [30,90) }");
+  CheckExpectedRanges(DemuxerStream::VIDEO, "{ [70,90) }");
+  CheckExpectedRanges("{ [70,90) }");
+  CheckExpectedBuffers(audio_stream, "30K 40K 50K 60K 70K 80K");
+  CheckExpectedBuffers(video_stream, "70K 80");
+}
+
+TEST_F(ChunkDemuxerTest,
+       RelaxedKeyframe_RemoveInterruptsMuxedCodedFrameGroup_2) {
+  // Append muxed:
+  //        A:[K K K]
+  //        V:(Nothing, simulating jagged cluster start or a long previous
+  //          video frame)
+  // Remove    *****
+  // Append muxed:
+  //        A:       [K K K][K K K]
+  //        V:       [n n n][n K n]
+  // Expect:
+  //        A:       [K K K][K K K]
+  //        V [................K n] (As would occur if there really were a
+  //        jagged cluster start and not badly muxed clusters as used to
+  //        simulate a jagged start in this test.)
+  ASSERT_TRUE(InitDemuxer(HAS_AUDIO | HAS_VIDEO));
+  DemuxerStream* audio_stream = demuxer_->GetStream(DemuxerStream::AUDIO);
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  EXPECT_MEDIA_LOG(SegmentMissingFrames("video"));
+  AppendMuxedCluster(MuxedStreamInfo(kAudioTrackNum, "0K 10K 20D10K"),
+                     MuxedStreamInfo(kVideoTrackNum, ""));
+  demuxer_->Remove(kSourceId, base::TimeDelta(),
+                   base::TimeDelta::FromMilliseconds(30));
+  AppendMuxedCluster(MuxedStreamInfo(kAudioTrackNum, "30K 40K 50D10K"),
+                     MuxedStreamInfo(kVideoTrackNum, "30 40 50", 10));
+  AppendMuxedCluster(MuxedStreamInfo(kAudioTrackNum, "60K 70K 80D10K"),
+                     MuxedStreamInfo(kVideoTrackNum, "60 70K 80", 10));
+  CheckExpectedRanges(DemuxerStream::AUDIO, "{ [30,90) }");
+  CheckExpectedRanges(DemuxerStream::VIDEO, "{ [0,90) }");
+  CheckExpectedRanges("{ [30,90) }");
+  CheckExpectedBuffers(audio_stream, "30K 40K 50K 60K 70K 80K");
+  CheckExpectedBuffers(video_stream, "70K 80");
+}
+
+TEST_F(ChunkDemuxerTest,
+       RelaxedKeyframe_RemoveInterruptsMuxedCodedFrameGroup_3) {
+  // Append muxed:
+  //        A:[K K K
+  //        V:(Nothing yet. This is a jagged start, not simulated.)
+  // Remove    *****
+  // Append muxed:
+  //        A:       K K K K K K]
+  //        V:       n n n n K n]
+  // Expect:
+  //        A:      [K K K K K K]
+  //        V [..............K n]
+  ASSERT_TRUE(InitDemuxer(HAS_AUDIO | HAS_VIDEO));
+  DemuxerStream* audio_stream = demuxer_->GetStream(DemuxerStream::AUDIO);
+  DemuxerStream* video_stream = demuxer_->GetStream(DemuxerStream::VIDEO);
+
+  std::vector<MuxedStreamInfo> msi(2);
+  msi[0] =
+      MuxedStreamInfo(kAudioTrackNum, "0K 10K 20K 30K 40K 50K 60K 70K 80D10K");
+  msi[1] = MuxedStreamInfo(kVideoTrackNum, "31 41 51 61 71K 81", 10);
+  scoped_ptr<Cluster> cluster = GenerateMuxedCluster(msi);
+
+  // Append the first part of the cluster, up to the beginning of the first
+  // video simpleblock. The result should be just 4 audio blocks and no video
+  // blocks are appended. Since the stream parser does not yet have a duration
+  // for the 4th audio block in this partial cluster append, it is not yet
+  // emitted from the parser, and only the first 3 audio blocks are expected to
+  // be buffered by and available from the demuxer.
+  ASSERT_EQ(kVideoTrackNum, 1);
+  int video_start = 0;
+  bool found = false;
+  while (video_start < cluster->size() - 10) {
+    if (cluster->data()[video_start] == 0xA3 &&
+        cluster->data()[video_start + 9] == 0x81) {
+      found = true;
+      break;
+    }
+    video_start++;
+  }
+
+  ASSERT_TRUE(found);
+  ASSERT_GT(video_start, 0);
+  ASSERT_LT(video_start, cluster->size() - 3);
+
+  AppendData(kSourceId, cluster->data(), video_start);
+  CheckExpectedRanges(DemuxerStream::AUDIO, "{ [0,30) }");
+  CheckExpectedRanges(DemuxerStream::VIDEO, "{ }");
+
+  demuxer_->Remove(kSourceId, base::TimeDelta(),
+                   base::TimeDelta::FromMilliseconds(30));
+
+  // Append the remainder of the cluster
+  AppendData(kSourceId, cluster->data() + video_start,
+             cluster->size() - video_start);
+
+  CheckExpectedRanges(DemuxerStream::AUDIO, "{ [30,90) }");
+  CheckExpectedRanges(DemuxerStream::VIDEO, "{ [0,91) }");
+  CheckExpectedRanges("{ [30,90) }");
+  CheckExpectedBuffers(audio_stream, "30K 40K 50K 60K 70K 80K");
+  CheckExpectedBuffers(video_stream, "71K 81");
+}
+
 }  // namespace media