Cast: improve AV sync in sender application

media/cast/test/sender.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Test application that simulates a cast sender - Data can be either generated
 // or read from a file.
 
 #include <queue>
 
 #include "base/at_exit.h"
@@ skipping 137 matching lines @@
         synthetic_count_(0),
         clock_(clock),
         audio_frame_count_(0),
         video_frame_count_(0),
         weak_factory_(this),
         av_format_context_(NULL),
         audio_stream_index_(-1),
         playback_rate_(1.0),
         video_stream_index_(-1),
         video_frame_rate_numerator_(video_config.max_frame_rate),
-        video_frame_rate_denominator_(1) {
+        video_frame_rate_denominator_(1),
+        video_first_pts_(0),
+        video_first_pts_set_(false) {
     audio_bus_factory_.reset(new TestAudioBusFactory(kAudioChannels,
                                                      kAudioSamplingFrequency,
                                                      kSoundFrequency,
                                                      kSoundVolume));
     const CommandLine* cmd = CommandLine::ForCurrentProcess();
     int override_fps = 0;
     if (base::StringToInt(cmd->GetSwitchValueASCII(kSwitchFps),
                           &override_fps)) {
       video_config_.max_frame_rate = override_fps;
       video_frame_rate_numerator_ = override_fps;
@@ skipping 112 matching lines @@
   void Start(scoped_refptr<AudioFrameInput> audio_frame_input,
              scoped_refptr<VideoFrameInput> video_frame_input) {
     audio_frame_input_ = audio_frame_input;
     video_frame_input_ = video_frame_input;
 
     LOG(INFO) << "Max Frame rate: " << video_config_.max_frame_rate;
     LOG(INFO) << "Real Frame rate: "
               << video_frame_rate_numerator_ << "/"
               << video_frame_rate_denominator_ << " fps.";
     LOG(INFO) << "Audio playback rate: " << playback_rate_;
+
+    if (!is_transcoding_audio() && !is_transcoding_video()) {
+      // Send fake patterns.
+      test_app_thread_proxy_->PostTask(
+          FROM_HERE,
+          base::Bind(
+              &SendProcess::SendNextFakeFrame,
+              base::Unretained(this)));
+      return;
+    }
+
+    // Send transcoding streams.
     audio_algo_.Initialize(playback_rate_, audio_params_);
     audio_algo_.FlushBuffers();
     audio_fifo_input_bus_ =
         AudioBus::Create(
             audio_params_.channels(), audio_params_.frames_per_buffer());
     // Audio FIFO can carry all data fron AudioRendererAlgorithm.
     audio_fifo_.reset(
         new AudioFifo(audio_params_.channels(),
                       audio_algo_.QueueCapacity()));
     audio_resampler_.reset(new media::MultiChannelResampler(
         audio_params_.channels(),
         static_cast<double>(audio_params_.sample_rate()) /
         kAudioSamplingFrequency,
         audio_params_.frames_per_buffer(),
         base::Bind(&SendProcess::ProvideData, base::Unretained(this))));
-    audio_decoded_ts_.reset(
-        new AudioTimestampHelper(audio_params_.sample_rate()));
-    audio_decoded_ts_->SetBaseTimestamp(base::TimeDelta());
-    audio_scaled_ts_.reset(
-        new AudioTimestampHelper(audio_params_.sample_rate()));
-    audio_scaled_ts_->SetBaseTimestamp(base::TimeDelta());
-    audio_resampled_ts_.reset(
-        new AudioTimestampHelper(kAudioSamplingFrequency));
-    audio_resampled_ts_->SetBaseTimestamp(base::TimeDelta());
     test_app_thread_proxy_->PostTask(
         FROM_HERE,
-        base::Bind(&SendProcess::SendNextFrame, base::Unretained(this)));
+        base::Bind(
+            &SendProcess::SendNextFrame,
+            base::Unretained(this)));
   }
 
-  void SendNextFrame() {
+  void SendNextFakeFrame() {
     gfx::Size size(video_config_.width, video_config_.height);
     scoped_refptr<VideoFrame> video_frame =
         VideoFrame::CreateBlackFrame(size);
-    if (is_transcoding_video()) {
-      Decode(false);
-      CHECK(!video_frame_queue_.empty()) << "No video frame.";
-      scoped_refptr<VideoFrame> decoded_frame =
-          video_frame_queue_.front();
-      video_frame->set_timestamp(decoded_frame->timestamp());
-      video_frame_queue_.pop();
-      media::CopyPlane(VideoFrame::kYPlane,
-                       decoded_frame->data(VideoFrame::kYPlane),
-                       decoded_frame->stride(VideoFrame::kYPlane),
-                       decoded_frame->rows(VideoFrame::kYPlane),
-                       video_frame);
-      media::CopyPlane(VideoFrame::kUPlane,
-                       decoded_frame->data(VideoFrame::kUPlane),
-                       decoded_frame->stride(VideoFrame::kUPlane),
-                       decoded_frame->rows(VideoFrame::kUPlane),
-                       video_frame);
-      media::CopyPlane(VideoFrame::kVPlane,
-                       decoded_frame->data(VideoFrame::kVPlane),
-                       decoded_frame->stride(VideoFrame::kVPlane),
-                       decoded_frame->rows(VideoFrame::kVPlane),
-                       video_frame);
-    } else {
-      PopulateVideoFrame(video_frame, synthetic_count_);
-    }
+    PopulateVideoFrame(video_frame, synthetic_count_);
     ++synthetic_count_;
 
     base::TimeTicks now = clock_->NowTicks();
     if (start_time_.is_null())
       start_time_ = now;
 
-    base::TimeDelta video_time;
-    if (is_transcoding_video()) {
-      // Use the timestamp from the file if we're transcoding and
-      // playback rate is 1.0.
-      video_time = ScaleTimestamp(video_frame->timestamp());
-    } else {
-      VideoFrameTime(video_frame_count_);
-    }
-
+    base::TimeDelta video_time = VideoFrameTime(video_frame_count_);
     video_frame->set_timestamp(video_time);
     video_frame_input_->InsertRawVideoFrame(video_frame,
                                             start_time_ + video_time);
 
-    if (is_transcoding_video()) {
-      // Decode next video frame to get the next frame's timestamp.
-      Decode(false);
-      CHECK(!video_frame_queue_.empty()) << "No video frame.";
-      video_time = ScaleTimestamp(video_frame_queue_.front()->timestamp());
-    } else {
-      video_time = VideoFrameTime(++video_frame_count_);
-    }
-
     // Send just enough audio data to match next video frame's time.
     base::TimeDelta audio_time = AudioFrameTime(audio_frame_count_);
     while (audio_time < video_time) {
       if (is_transcoding_audio()) {
         Decode(true);
         CHECK(!audio_bus_queue_.empty()) << "No audio decoded.";
         scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
         audio_bus_queue_.pop();
         audio_frame_input_->InsertAudio(
             bus.Pass(), start_time_ + audio_time);
       } else {
         audio_frame_input_->InsertAudio(
             audio_bus_factory_->NextAudioBus(
                 base::TimeDelta::FromMilliseconds(kAudioFrameMs)),
             start_time_ + audio_time);
       }
       audio_time = AudioFrameTime(++audio_frame_count_);
     }
 
     // This is the time since the stream started.
     const base::TimeDelta elapsed_time = now - start_time_;
 
-    // Handle the case when decoding or frame generation cannot keep up.
+    // Handle the case when frame generation cannot keep up.
     // Move the time ahead to match the next frame.
     while (video_time < elapsed_time) {
       LOG(WARNING) << "Skipping one frame.";
       video_time = VideoFrameTime(++video_frame_count_);
     }
 
     test_app_thread_proxy_->PostDelayedTask(
         FROM_HERE,
-        base::Bind(&SendProcess::SendNextFrame,
+        base::Bind(&SendProcess::SendNextFakeFrame,
                    weak_factory_.GetWeakPtr()),
         video_time - elapsed_time);
   }
 
+  // Return true if a frame was sent.
+  bool SendNextTranscodedVideo(base::TimeDelta elapsed_time) {
+    if (!is_transcoding_video())
+      return false;
+
+    Decode(false);
+    if (video_frame_queue_.empty())
+      return false;
+
+    scoped_refptr<VideoFrame> decoded_frame =
+        video_frame_queue_.front();
+    if (elapsed_time < decoded_frame->timestamp())
+      return false;
+
+    gfx::Size size(video_config_.width, video_config_.height);
+    scoped_refptr<VideoFrame> video_frame =
+        VideoFrame::CreateBlackFrame(size);
+    video_frame_queue_.pop();
+    media::CopyPlane(VideoFrame::kYPlane,
+                     decoded_frame->data(VideoFrame::kYPlane),
+                     decoded_frame->stride(VideoFrame::kYPlane),
+                     decoded_frame->rows(VideoFrame::kYPlane),
+                     video_frame);
+    media::CopyPlane(VideoFrame::kUPlane,
+                     decoded_frame->data(VideoFrame::kUPlane),
+                     decoded_frame->stride(VideoFrame::kUPlane),
+                     decoded_frame->rows(VideoFrame::kUPlane),
+                     video_frame);
+    media::CopyPlane(VideoFrame::kVPlane,
+                     decoded_frame->data(VideoFrame::kVPlane),
+                     decoded_frame->stride(VideoFrame::kVPlane),
+                     decoded_frame->rows(VideoFrame::kVPlane),
+                     video_frame);
+
+    base::TimeDelta video_time;
+    // Use the timestamp from the file if we're transcoding and
+    // playback rate is 1.0.

[hubbe, 2014/04/24 22:32:51]

Odd comment, what do we use if playback rate is not 1.0?

+    video_time = ScaleTimestamp(decoded_frame->timestamp());
+    video_frame_input_->InsertRawVideoFrame(
+        video_frame, start_time_ + video_time);
+
+    // Make sure queue is not empty.
+    Decode(false);
+    return true;
+  }
+
+  // Return true if a frame was sent.
+  bool SendNextTranscodedAudio(base::TimeDelta elapsed_time) {
+    if (!is_transcoding_audio())
+      return false;
+
+    Decode(true);
+    if (audio_bus_queue_.empty())
+      return false;
+
+    base::TimeDelta audio_time = audio_sent_ts_->GetTimestamp();
+    if (elapsed_time < audio_time)
+      return false;
+    scoped_ptr<AudioBus> bus(audio_bus_queue_.front());
+    audio_bus_queue_.pop();
+    audio_sent_ts_->AddFrames(bus->frames());
+    audio_frame_input_->InsertAudio(
+        bus.Pass(), start_time_ + audio_time);
+
+    // Make sure queue is not empty.
+    Decode(true);
+    return true;
+  }
+
+  void SendNextFrame() {
+    if (start_time_.is_null())
+      start_time_ = clock_->NowTicks();
+    if (start_time_.is_null())
+      start_time_ = clock_->NowTicks();
+
+    // Send as much as possible. Audio is sent according to
+    // system time.
+    while (SendNextTranscodedAudio(
+               clock_->NowTicks() - start_time_));

[hubbe, 2014/04/24 22:32:51]

weird indentation.

+    // Video is sync'ed to audio.
+    while (SendNextTranscodedVideo(
+               audio_sent_ts_->GetTimestamp()));

[hubbe, 2014/04/24 22:32:51]

weird indentation

+
+    if (audio_bus_queue_.empty() && video_frame_queue_.empty()) {
+      // Both queues are empty can only mean that we have reached
+      // the end of the stream.
+      LOG(INFO) << "Rewind.";
+      Rewind();
+      start_time_ = base::TimeTicks();
+      audio_sent_ts_.reset();
+      video_first_pts_set_ = false;
+    }
+
+    // Send next send.
+    test_app_thread_proxy_->PostDelayedTask(
+        FROM_HERE,
+        base::Bind(
+            &SendProcess::SendNextFrame,
+            base::Unretained(this)),
+        base::TimeDelta::FromMilliseconds(kAudioFrameMs));
+  }
+
   const VideoSenderConfig& get_video_config() const { return video_config_; }
 
  private:
   bool is_transcoding_audio() { return audio_stream_index_ >= 0; }
   bool is_transcoding_video() { return video_stream_index_ >= 0; }
 
   // Helper methods to compute timestamps for the frame number specified.
   base::TimeDelta VideoFrameTime(int frame_number) {
     return frame_number * base::TimeDelta::FromSeconds(1) *
         video_frame_rate_denominator_ / video_frame_rate_numerator_;
@@ skipping 11 matching lines @@
   // Go to the beginning of the stream.
   void Rewind() {
     CHECK(av_seek_frame(av_format_context_, -1, 0, AVSEEK_FLAG_BACKWARD) >= 0)
         << "Failed to rewind to the beginning.";
   }
 
   // Call FFmpeg to fetch one packet.
   ScopedAVPacket DemuxOnePacket(bool* audio) {
     ScopedAVPacket packet(new AVPacket());
     if (av_read_frame(av_format_context_, packet.get()) < 0) {
-      LOG(ERROR) << "Failed to read one AVPacket";
+      LOG(ERROR) << "Failed to read one AVPacket.";
       packet.reset();
       return packet.Pass();
     }
 
     int stream_index = static_cast<int>(packet->stream_index);
     if (stream_index == audio_stream_index_) {
       *audio = true;
     } else if (stream_index == video_stream_index_) {
       *audio = false;
     } else {
@@ skipping 19 matching lines @@
       CHECK(result >= 0) << "Failed to decode audio.";
       packet_temp.size -= result;
       packet_temp.data += result;
       if (!frame_decoded)
         continue;
 
       int frames_read = avframe->nb_samples;
       if (frames_read < 0)
         break;
 
+      if (!audio_sent_ts_) {
+        // Initialize the base time to the first packet in the file.
+        // This is set to the frequency we send to the receiver.
+        // Not the frequency of the source file. This is because we
+        // increment the frame count by samples we sent.
+        audio_sent_ts_.reset(
+            new AudioTimestampHelper(kAudioSamplingFrequency));
+        // For some files this is an invalid value.
+        base::TimeDelta base_ts;
+        audio_sent_ts_->SetBaseTimestamp(base_ts);
+      }
+
       scoped_refptr<AudioBuffer> buffer =
           AudioBuffer::CopyFrom(
               AVSampleFormatToSampleFormat(
                   av_audio_context()->sample_fmt),
               ChannelLayoutToChromeChannelLayout(
                   av_audio_context()->channel_layout,
                   av_audio_context()->channels),
               av_audio_context()->channels,
               av_audio_context()->sample_rate,
               frames_read,
               &avframe->data[0],
-              audio_decoded_ts_->GetTimestamp(),
-              audio_decoded_ts_->GetFrameDuration(frames_read));
+              // Note: Not all files have correct values for pkt_pts.
+              base::TimeDelta::FromMilliseconds(avframe->pkt_pts),
+              // TODO(hclam): Give accurate duration based on samples.
+              base::TimeDelta());
       audio_algo_.EnqueueBuffer(buffer);
-      audio_decoded_ts_->AddFrames(frames_read);
     } while (packet_temp.size > 0);
     avcodec_free_frame(&avframe);
 
     const int frames_needed_to_scale =
         playback_rate_ * av_audio_context()->sample_rate /
         kAudioPacketsPerSecond;
     while (frames_needed_to_scale <= audio_algo_.frames_buffered()) {
       if (!audio_algo_.FillBuffer(audio_fifo_input_bus_.get(),
                                   audio_fifo_input_bus_->frames())) {
         // Nothing can be scaled. Decode some more.
         return;
       }
-      audio_scaled_ts_->AddFrames(audio_fifo_input_bus_->frames());
 
       // Prevent overflow of audio data in the FIFO.
       if (audio_fifo_input_bus_->frames() + audio_fifo_->frames()
           <= audio_fifo_->max_frames()) {
         audio_fifo_->Push(audio_fifo_input_bus_.get());
       } else {
         LOG(WARNING) << "Audio FIFO full; dropping samples.";
       }
 
       // Make sure there's enough data to resample audio.
       if (audio_fifo_->frames() <
           2 * audio_params_.sample_rate() / kAudioPacketsPerSecond) {
         continue;
       }
 
       scoped_ptr<media::AudioBus> resampled_bus(
           media::AudioBus::Create(
               audio_params_.channels(),
               kAudioSamplingFrequency / kAudioPacketsPerSecond));
       audio_resampler_->Resample(resampled_bus->frames(),
                                  resampled_bus.get());
-      audio_resampled_ts_->AddFrames(resampled_bus->frames());
       audio_bus_queue_.push(resampled_bus.release());
     }
   }
 
   void DecodeVideo(ScopedAVPacket packet) {
     // Video.
     int got_picture;
     AVFrame* avframe = av_frame_alloc();
     avcodec_get_frame_defaults(avframe);
     // Tell the decoder to reorder for us.
     avframe->reordered_opaque =
         av_video_context()->reordered_opaque = packet->pts;
     CHECK(avcodec_decode_video2(
         av_video_context(), avframe, &got_picture, packet.get()) >= 0)
         << "Video decode error.";
     if (!got_picture)
       return;
     gfx::Size size(av_video_context()->width, av_video_context()->height);
+    if (!video_first_pts_set_ ||
+        avframe->reordered_opaque < video_first_pts_) {
+      video_first_pts_set_ = true;
+      video_first_pts_ = avframe->reordered_opaque;
+    }
+    int64 pts = avframe->reordered_opaque - video_first_pts_;
     video_frame_queue_.push(
         VideoFrame::WrapExternalYuvData(
             media::VideoFrame::YV12,
             size,
             gfx::Rect(size),
             size,
             avframe->linesize[0],
             avframe->linesize[1],
             avframe->linesize[2],
             avframe->data[0],
             avframe->data[1],
             avframe->data[2],
-            base::TimeDelta::FromMilliseconds(avframe->reordered_opaque),
+            base::TimeDelta::FromMilliseconds(pts),
             base::Bind(&AVFreeFrame, avframe)));
   }
 
   void Decode(bool decode_audio) {
     // Read the stream until one video frame can be decoded.
     while (true) {
       if (decode_audio && !audio_bus_queue_.empty())
         return;
       if (!decode_audio && !video_frame_queue_.empty())
         return;
 
       bool audio_packet = false;
       ScopedAVPacket packet = DemuxOnePacket(&audio_packet);
       if (!packet) {
-        LOG(INFO) << "End of stream; Rewind.";
-        Rewind();
-        continue;
+        LOG(INFO) << "End of stream.";
+        return;
       }
 
       if (audio_packet)
         DecodeAudio(packet.Pass());
       else
         DecodeVideo(packet.Pass());
     }
   }
 
   void ProvideData(int frame_delay, media::AudioBus* output_bus) {
@@ skipping 13 matching lines @@
   }
   AVCodecContext* av_audio_context() { return av_audio_stream()->codec; }
   AVCodecContext* av_video_context() { return av_video_stream()->codec; }
 
   scoped_refptr<base::SingleThreadTaskRunner> test_app_thread_proxy_;
   VideoSenderConfig video_config_;
   scoped_refptr<AudioFrameInput> audio_frame_input_;
   scoped_refptr<VideoFrameInput> video_frame_input_;
   uint8 synthetic_count_;
   base::TickClock* const clock_;  // Not owned by this class.
+
+  // Time when the stream starts.
   base::TimeTicks start_time_;
+
+  // The following three members are used only for fake frames.
   int audio_frame_count_;  // Each audio frame is exactly 10ms.
   int video_frame_count_;
   scoped_ptr<TestAudioBusFactory> audio_bus_factory_;
 
   // NOTE: Weak pointers must be invalidated before all other member variables.
   base::WeakPtrFactory<SendProcess> weak_factory_;
 
   base::MemoryMappedFile file_data_;
   scoped_ptr<InMemoryUrlProtocol> protocol_;
   scoped_ptr<FFmpegGlue> glue_;
   AVFormatContext* av_format_context_;
 
   int audio_stream_index_;
   AudioParameters audio_params_;
   double playback_rate_;
 
   int video_stream_index_;
   int video_frame_rate_numerator_;
   int video_frame_rate_denominator_;
 
   // These are used for audio resampling.
   scoped_ptr<media::MultiChannelResampler> audio_resampler_;
   scoped_ptr<media::AudioFifo> audio_fifo_;
   scoped_ptr<media::AudioBus> audio_fifo_input_bus_;
   media::AudioRendererAlgorithm audio_algo_;
 
-  // These helpers are used to track frames generated.
-  // They are:
-  // * Frames decoded from the file.
-  // * Frames scaled according to playback rate.
-  // * Frames resampled to output frequency.
-  scoped_ptr<media::AudioTimestampHelper> audio_decoded_ts_;
-  scoped_ptr<media::AudioTimestampHelper> audio_scaled_ts_;
-  scoped_ptr<media::AudioTimestampHelper> audio_resampled_ts_;
+  // Track the timestamp of audio sent to the receiver.
+  scoped_ptr<media::AudioTimestampHelper> audio_sent_ts_;
 
   std::queue<scoped_refptr<VideoFrame> > video_frame_queue_;
+  int64 video_first_pts_;
+  bool video_first_pts_set_;
+
   std::queue<AudioBus*> audio_bus_queue_;
 
   DISALLOW_COPY_AND_ASSIGN(SendProcess);
 };
 
 }  // namespace cast
 }  // namespace media
 
 namespace {
 void UpdateCastTransportStatus(
@@ skipping 217 matching lines @@
                  base::Passed(&video_event_subscriber),
                  base::Passed(&audio_event_subscriber),
                  base::Passed(&video_log_file),
                  base::Passed(&audio_log_file)),
       base::TimeDelta::FromSeconds(logging_duration_seconds));
   send_process->Start(cast_sender->audio_frame_input(),
                       cast_sender->video_frame_input());
   io_message_loop.Run();
   return 0;
 }