MediaCodecPlayer implementation (stage 5 - reconfiguration)

media/base/android/media_codec_player_unittest.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/timer/timer.h"
 #include "media/base/android/demuxer_android.h"
 #include "media/base/android/media_codec_bridge.h"
 #include "media/base/android/media_codec_player.h"
@@ skipping 44 matching lines @@
   return (a - b).magnitude().InMilliseconds() <= tolerance_ms;
 }
 
 // Mock of MediaPlayerManager for testing purpose.
 
 class MockMediaPlayerManager : public MediaPlayerManager {
  public:
   MockMediaPlayerManager()
       : playback_completed_(false),
         num_seeks_completed_(0),
+        num_audio_codecs_created_(0),
+        num_video_codecs_created_(0),
         weak_ptr_factory_(this) {}
   ~MockMediaPlayerManager() override {}
 
   MediaResourceGetter* GetMediaResourceGetter() override { return nullptr; }
   MediaUrlInterceptor* GetMediaUrlInterceptor() override { return nullptr; }
 
   // Regular time update callback, reports current playback time to
   // MediaPlayerManager.
   void OnTimeUpdate(int player_id,
                     base::TimeDelta current_timestamp,
@@ skipping 30 matching lines @@
   MediaPlayerAndroid* GetPlayer(int player_id) override { return nullptr; }
   bool RequestPlay(int player_id) override { return true; }
 
   void OnMediaResourcesRequested(int player_id) {}
 
   // Time update callback that reports the internal progress of the stream.
   // Implementation dependent, used for testing only.
   void OnDecodersTimeUpdate(DemuxerStream::Type stream_type,
                             base::TimeDelta now_playing,
                             base::TimeDelta last_buffered) {
-    PTSTime& hit = first_frame_hit_[stream_type];
-    if (hit.is_null())
-      hit = PTSTime(now_playing, base::TimeTicks::Now());
+    render_stat_[stream_type].AddValue(
+        PTSTime(now_playing, base::TimeTicks::Now()));
+  }
+
+  // Notification called on MediaCodec creation.
+  // Implementation dependent, used for testing only.
+  void OnMediaCodecCreated(DemuxerStream::Type stream_type) {
+    if (stream_type == DemuxerStream::AUDIO)
+      ++num_audio_codecs_created_;
+    else if (stream_type == DemuxerStream::VIDEO)
+      ++num_video_codecs_created_;
   }
 
   // First frame information
   base::TimeDelta FirstFramePTS(DemuxerStream::Type stream_type) const {
-    return first_frame_hit_[stream_type].pts;
+    return render_stat_[stream_type].min().pts;
   }
   base::TimeTicks FirstFrameTime(DemuxerStream::Type stream_type) const {
-    return first_frame_hit_[stream_type].time;
+    return render_stat_[stream_type].min().time;
   }
 
   base::WeakPtr<MockMediaPlayerManager> GetWeakPtr() {
     return weak_ptr_factory_.GetWeakPtr();
   }
 
   // Conditions to wait for.
   bool IsMetadataChanged() const { return media_metadata_.modified; }
   bool IsPlaybackCompleted() const { return playback_completed_; }
   bool IsPlaybackStarted() const { return pts_stat_.num_values() > 0; }
   bool IsPlaybackBeyondPosition(const base::TimeDelta& pts) const {
     return pts_stat_.max() > pts;
   }
   bool IsSeekCompleted() const { return num_seeks_completed_ > 0; }
   bool HasFirstFrame(DemuxerStream::Type stream_type) const {
-    return !first_frame_hit_[stream_type].is_null();
+    return render_stat_[stream_type].num_values() != 0;
   }
 
+  int num_audio_codecs_created() const { return num_audio_codecs_created_; }
+  int num_video_codecs_created() const { return num_video_codecs_created_; }
+
   struct MediaMetadata {
     base::TimeDelta duration;
     int width;
     int height;
     bool modified;
     MediaMetadata() : width(0), height(0), modified(false) {}
   };
   MediaMetadata media_metadata_;
 
+  struct PTSTime {
+    base::TimeDelta pts;
+    base::TimeTicks time;
+
+    PTSTime() : pts(), time() {}
+    PTSTime(base::TimeDelta p, base::TimeTicks t) : pts(p), time(t) {}
+    bool is_null() const { return time.is_null(); }
+    bool operator<(const PTSTime& rhs) const { return time < rhs.time; }
+  };
+  Minimax<PTSTime> render_stat_[DemuxerStream::NUM_TYPES];
+
   Minimax<base::TimeDelta> pts_stat_;
 
  private:
   bool playback_completed_;
   int num_seeks_completed_;
-
-  struct PTSTime {
-    base::TimeDelta pts;
-    base::TimeTicks time;
-    PTSTime() : pts(), time() {}
-    PTSTime(base::TimeDelta p, base::TimeTicks t) : pts(p), time(t) {}
-    bool is_null() const { return time.is_null(); }
-  };
-  PTSTime first_frame_hit_[DemuxerStream::NUM_TYPES];
+  int num_audio_codecs_created_;
+  int num_video_codecs_created_;
 
   base::WeakPtrFactory<MockMediaPlayerManager> weak_ptr_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(MockMediaPlayerManager);
 };
 
 // Helper method that creates demuxer configuration.
 
 DemuxerConfigs CreateAudioVideoConfigs(const base::TimeDelta& duration,
                                        const gfx::Size& video_size) {
@@ skipping 198 matching lines @@
 
   bool created = false;
   if (type == DemuxerStream::AUDIO && audio_factory_)
     created = audio_factory_->CreateChunk(&chunk, &delay);
   else if (type == DemuxerStream::VIDEO && video_factory_)
     created = video_factory_->CreateChunk(&chunk, &delay);
 
   if (!created)
     return;
 
+  // Request key frame after |kConfigChanged|
+  if (type == DemuxerStream::VIDEO && !chunk.demuxer_configs.empty())
+    video_factory_->RequestKeyFrame();
+
   chunk.type = type;
 
   // Post to the Media thread. Use the weak pointer to prevent the data arrival
   // after the player has been deleted.
   GetMediaTaskRunner()->PostDelayedTask(
       FROM_HERE, base::Bind(&MockDemuxerAndroid::OnDemuxerDataAvailable,
                             weak_factory_.GetWeakPtr(), chunk),
       delay);
 }
 
@@ skipping 1162 matching lines @@
 
   // Post configuration after the player has been initialized.
   demuxer_->PostInternalConfigs();
 
   // Issue SeekTo().
   player_->SeekTo(seek_position);
 
   // Start the playback.
   player_->Start();
 
-  // The video decoder should start prerolling
+  // The video decoder should start prerolling.
   // Wait till preroll starts.
   EXPECT_TRUE(WaitForCondition(
       base::Bind(&MediaCodecPlayer::IsPrerollingForTests,
                  base::Unretained(player_), DemuxerStream::VIDEO),
       start_timeout));
 
   // Wait for playback to start.
   EXPECT_TRUE(
       WaitForCondition(base::Bind(&MockMediaPlayerManager::IsPlaybackStarted,
                                   base::Unretained(&manager_)),
                        preroll_timeout));
 
   EXPECT_TRUE(manager_.HasFirstFrame(DemuxerStream::AUDIO));
 
   // Play till completion.
   EXPECT_TRUE(
       WaitForCondition(base::Bind(&MockMediaPlayerManager::IsPlaybackCompleted,
                                   base::Unretained(&manager_))));
 
   // There should not be any video frames.
   EXPECT_FALSE(manager_.HasFirstFrame(DemuxerStream::VIDEO));
 }
 
+TEST_F(MediaCodecPlayerTest, AVVideoConfigChangeWhilePlaying) {
+  SKIP_TEST_IF_MEDIA_CODEC_BRIDGE_IS_NOT_AVAILABLE();
+
+  // Test that playback continues after video config change.
+
+  // Initialize A/V playback
+  base::TimeDelta duration = base::TimeDelta::FromMilliseconds(1600);
+  base::TimeDelta config_change_position =
+      base::TimeDelta::FromMilliseconds(1000);
+
+  base::TimeDelta start_timeout = base::TimeDelta::FromMilliseconds(2000);
+  base::TimeDelta completion_timeout = base::TimeDelta::FromMilliseconds(3000);
+
+  demuxer_->SetAudioFactory(
+      scoped_ptr<AudioFactory>(new AudioFactory(duration)));
+  demuxer_->SetVideoFactory(
+      scoped_ptr<VideoFactory>(new VideoFactory(duration)));
+
+  demuxer_->video_factory()->RequestConfigChange(config_change_position);
+
+  CreatePlayer();
+  SetVideoSurface();
+
+  // Wait till the player is initialized on media thread.
+  EXPECT_TRUE(WaitForCondition(base::Bind(&MockDemuxerAndroid::IsInitialized,
+                                          base::Unretained(demuxer_))));
+
+  if (!demuxer_->IsInitialized()) {
+    DVLOG(0) << "AVConfigChangeWhilePlaying: demuxer is not initialized";
+    return;
+  }
+
+  // Ask decoders to always reconfigure after the player has been initialized.
+  player_->SetAlwaysReconfigureForTests(DemuxerStream::VIDEO);
+
+  // Set a testing callback to receive PTS from decoders.
+  player_->SetDecodersTimeCallbackForTests(
+      base::Bind(&MockMediaPlayerManager::OnDecodersTimeUpdate,
+                 base::Unretained(&manager_)));
+
+  // Set a testing callback to receive MediaCodec creation events from decoders.
+  player_->SetCodecCreatedCallbackForTests(
+      base::Bind(&MockMediaPlayerManager::OnMediaCodecCreated,
+                 base::Unretained(&manager_)));
+
+  // Post configuration after the player has been initialized.
+  demuxer_->PostInternalConfigs();
+
+  // Start and wait for playback.
+  player_->Start();
+
+  // Wait till we start to play.
+  EXPECT_TRUE(
+      WaitForCondition(base::Bind(&MockMediaPlayerManager::IsPlaybackStarted,
+                                  base::Unretained(&manager_)),
+                       start_timeout));
+
+  // Wait another 100 ms to make sure we are done with initial preroll.
+  WaitForDelay(base::TimeDelta::FromMilliseconds(100));
+
+  EXPECT_FALSE(player_->IsPrerollingForTests(DemuxerStream::AUDIO));
+  EXPECT_FALSE(player_->IsPrerollingForTests(DemuxerStream::VIDEO));
+
+  // Wait till completion
+  EXPECT_TRUE(
+      WaitForCondition(base::Bind(&MockMediaPlayerManager::IsPlaybackCompleted,
+                                  base::Unretained(&manager_)),
+                       completion_timeout));
+
+  // The audio codec should be kept.
+  EXPECT_EQ(1, manager_.num_audio_codecs_created());
+
+  // The video codec should be recreated upon config changes.
+  EXPECT_EQ(2, manager_.num_video_codecs_created());
+
+  // Check that we did not miss video frames
+  int expected_video_frames = duration / kVideoFramePeriod + 1;
+  EXPECT_EQ(expected_video_frames,
+            manager_.render_stat_[DemuxerStream::VIDEO].num_values());
+
+  // Check that we did not miss audio frames. We expect two postponed frames
+  // that are not reported.
+  // For Nexus 4 KitKat the AAC decoder seems to swallow the first frame
+  // but reports the last pts twice, maybe it just shifts the reported PTS.
+  int expected_audio_frames = duration / kAudioFramePeriod + 1 - 2;
+  EXPECT_EQ(expected_audio_frames,
+            manager_.render_stat_[DemuxerStream::AUDIO].num_values());
+}
+
+TEST_F(MediaCodecPlayerTest, AVAudioConfigChangeWhilePlaying) {
+  SKIP_TEST_IF_MEDIA_CODEC_BRIDGE_IS_NOT_AVAILABLE();
+
+  // Test that playback continues after audio config change.
+
+  // Initialize A/V playback
+  base::TimeDelta duration = base::TimeDelta::FromMilliseconds(1600);
+  base::TimeDelta config_change_position =
+      base::TimeDelta::FromMilliseconds(1000);
+
+  base::TimeDelta start_timeout = base::TimeDelta::FromMilliseconds(2000);
+  base::TimeDelta completion_timeout = base::TimeDelta::FromMilliseconds(3000);
+
+  demuxer_->SetAudioFactory(
+      scoped_ptr<AudioFactory>(new AudioFactory(duration)));
+  demuxer_->SetVideoFactory(
+      scoped_ptr<VideoFactory>(new VideoFactory(duration)));
+
+  demuxer_->audio_factory()->RequestConfigChange(config_change_position);
+
+  CreatePlayer();
+  SetVideoSurface();
+
+  // Wait till the player is initialized on media thread.
+  EXPECT_TRUE(WaitForCondition(base::Bind(&MockDemuxerAndroid::IsInitialized,
+                                          base::Unretained(demuxer_))));
+
+  if (!demuxer_->IsInitialized()) {
+    DVLOG(0) << "AVConfigChangeWhilePlaying: demuxer is not initialized";
+    return;
+  }
+
+  // Ask decoders to always reconfigure after the player has been initialized.
+  player_->SetAlwaysReconfigureForTests(DemuxerStream::AUDIO);
+
+  // Set a testing callback to receive PTS from decoders.
+  player_->SetDecodersTimeCallbackForTests(
+      base::Bind(&MockMediaPlayerManager::OnDecodersTimeUpdate,
+                 base::Unretained(&manager_)));
+
+  // Set a testing callback to receive MediaCodec creation events from decoders.
+  player_->SetCodecCreatedCallbackForTests(
+      base::Bind(&MockMediaPlayerManager::OnMediaCodecCreated,
+                 base::Unretained(&manager_)));
+
+  // Post configuration after the player has been initialized.
+  demuxer_->PostInternalConfigs();
+
+  // Start and wait for playback.
+  player_->Start();
+
+  // Wait till we start to play.
+  EXPECT_TRUE(
+      WaitForCondition(base::Bind(&MockMediaPlayerManager::IsPlaybackStarted,
+                                  base::Unretained(&manager_)),
+                       start_timeout));
+
+  // Wait another 100 ms to make sure we are done with initial preroll.
+  WaitForDelay(base::TimeDelta::FromMilliseconds(100));

[liberato (no reviews please), 2015/08/27 20:42:26]

can this be converted to a WaitForCondition or callback?  100msec is pretty
short.

[Tima Vaisburd, 2015/08/27 22:04:19]

IsPlaybackStarted() comes when the first audio frame is rendered after preroll.
We wait for |start_timeout|, 2s, for this to happen. The first video frame might
come later, but the state should not be Prerolling. I will try to delete this
wait and see.

[Tima Vaisburd, 2015/08/28 00:16:48]

I removed the wait and the subsequent check for IsPrerollingForTests() in all
three tests altogether since the check do not add anything to the tests: we
check that decoder is recreated and that frames are not skipped.

+
+  EXPECT_FALSE(player_->IsPrerollingForTests(DemuxerStream::AUDIO));
+  EXPECT_FALSE(player_->IsPrerollingForTests(DemuxerStream::VIDEO));
+
+  // Wait till completion
+  EXPECT_TRUE(
+      WaitForCondition(base::Bind(&MockMediaPlayerManager::IsPlaybackCompleted,
+                                  base::Unretained(&manager_)),
+                       completion_timeout));
+
+  // The audio codec should be recreated upon config changes.
+  EXPECT_EQ(2, manager_.num_audio_codecs_created());
+
+  // The video codec should be kept.
+  EXPECT_EQ(1, manager_.num_video_codecs_created());
+
+  // Check that we did not miss video frames.
+  int expected_video_frames = duration / kVideoFramePeriod + 1;
+  EXPECT_EQ(expected_video_frames,
+            manager_.render_stat_[DemuxerStream::VIDEO].num_values());
+
+  // Check that we did not miss audio frames. We expect two postponed frames
+  // that are not reported.
+  int expected_audio_frames = duration / kAudioFramePeriod + 1 - 2;
+  EXPECT_EQ(expected_audio_frames,
+            manager_.render_stat_[DemuxerStream::AUDIO].num_values());
+}
+
 }  // namespace media