Mpeg2TS - estimate duration for video frames.

media/formats/mp2t/es_parser_h264.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/formats/mp2t/es_parser_h264.h"
 
 #include "base/basictypes.h"
 #include "base/logging.h"
 #include "base/numerics/safe_conversions.h"
 #include "media/base/buffers.h"
 #include "media/base/stream_parser_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/filters/h264_parser.h"
 #include "media/formats/common/offset_byte_queue.h"
 #include "media/formats/mp2t/mp2t_common.h"
 #include "ui/gfx/rect.h"
 #include "ui/gfx/size.h"
 
 namespace media {
 namespace mp2t {
 
 // An AUD NALU is at least 4 bytes:
 // 3 bytes for the start code + 1 byte for the NALU type.
 const int kMinAUDSize = 4;
 
+// Default video frame duration.
+const base::TimeDelta kDefaultFrameDuration(

[acolwell GONE FROM CHROMIUM, 2014/07/03 00:50:58]

nit: Please add a comment saying why you picked 25fps.

+    base::TimeDelta::FromMilliseconds(40));
+
+const size_t kMaxPtsHistoryLength = 16;

[acolwell GONE FROM CHROMIUM, 2014/07/03 00:50:58]

nit: I'm assuming this is derived based on the max number of B-frames that can
appear after non-B-frames right? Please add a comment indicating where this
number was derived from. 

+
 EsParserH264::EsParserH264(
     const NewVideoConfigCB& new_video_config_cb,
     const EmitBufferCB& emit_buffer_cb)
   : new_video_config_cb_(new_video_config_cb),
     emit_buffer_cb_(emit_buffer_cb),
     es_queue_(new media::OffsetByteQueue()),
     h264_parser_(new H264Parser()),
     current_access_unit_pos_(0),
     next_access_unit_pos_(0) {
 }
@@ skipping 42 matching lines @@
 }
 
 void EsParserH264::Reset() {
   DVLOG(1) << "EsParserH264::Reset";
   es_queue_.reset(new media::OffsetByteQueue());
   h264_parser_.reset(new H264Parser());
   current_access_unit_pos_ = 0;
   next_access_unit_pos_ = 0;
   timing_desc_list_.clear();
   last_video_decoder_config_ = VideoDecoderConfig();
+  frame_pts_history_.clear();
 }
 
 bool EsParserH264::FindAUD(int64* stream_pos) {
   while (true) {
     const uint8* es;
     int size;
     es_queue_->PeekAt(*stream_pos, &es, &size);
 
     // Find a start code and move the stream to the start code parser position.
     off_t start_code_offset;
@@ skipping 132 matching lines @@
   // Get the access unit timing info.
   TimingDesc current_timing_desc = {kNoTimestamp(), kNoTimestamp()};
   while (!timing_desc_list_.empty() &&
          timing_desc_list_.front().first <= access_unit_pos) {
     current_timing_desc = timing_desc_list_.front().second;
     timing_desc_list_.pop_front();
   }
   if (current_timing_desc.pts == kNoTimestamp())
     return false;
 
+  // In Mpeg2 TS, there is no notion of frame duration.
+  // Only try to get an estimate of the frame duration.
+  // TODO(damienv): when the H264 stream has SEI timing information,
+  // we could get more accurate information about the frame duration.
+  if (!frame_pts_history_.empty() &&
+      frame_pts_history_.back() > current_timing_desc.pts) {
+    // To estimate the frame duration, B frames are assigned the same PTS
+    // as the last non-B frame.
+    frame_pts_history_.push_back(frame_pts_history_.back());
+  } else {
+    frame_pts_history_.push_back(current_timing_desc.pts);
+  }
+  if (frame_pts_history_.size() > kMaxPtsHistoryLength)
+    frame_pts_history_.pop_front();
+
+  base::TimeDelta frame_duration = kDefaultFrameDuration;
+  int pts_count = frame_pts_history_.size();
+  if (pts_count > 2) {
+    base::TimeDelta avg_frame_duration =
+        (frame_pts_history_.back() - frame_pts_history_.front()) /
+        (pts_count - 1);
+    frame_duration =
+        ((kMaxPtsHistoryLength - pts_count) * kDefaultFrameDuration +

[acolwell GONE FROM CHROMIUM, 2014/07/03 00:50:58]

I'm assuming that you are trying to smooth out the value for the first couple
frames here right? Is that because you don't know how many B-frames there might
be and you don't know if hierarchical B-frames are being used? Perhaps a
comment?

It seems like you shouldn't have to estimate if you delay emitting frames until
you get to a non-B. I worry that incorrect estimates may cause problems
downstream if the pts + duration overlaps another frame.

[damienv1, 2014/07/07 17:26:34]

- In theory, to get the frame duration accurately, I would need to replicate
exactly the H264 frame bumping process here (which would involve a latency for
buffer emission, and that latency would be "optimal").
However, I don't think going that deep inside the H264 parsing is the right way
either (at least, it is not inline with the spirit of a demuxer which should not
have to parse ES streams in a first place).

- Delaying the frame buffer emission with a constant latency (i.e. not going
through the H264 frame bumping process) could be another way (also not as
generic). 

Incorrect frame duration estimates could impact the upper layer only if the
upper layer understands the decoding/rendering process (which shouldn't be the
intent of MSE).

I was not really in favor of introducing a latency in that part of the code
(when you have a muxed audio/video stream, the impact is the same as the video
was coming later into the stream: i.e. could make the stream become badly muxed
stream from a buffer emission perspective).

Let me think about it though.

+         pts_count * avg_frame_duration) / kMaxPtsHistoryLength;
+  }
+
   // Update the video decoder configuration if needed.
   const H264PPS* pps = h264_parser_->GetPPS(pps_id);
   if (!pps) {
     // Only accept an invalid PPS at the beginning when the stream
     // does not necessarily start with an SPS/PPS/IDR.
     // In this case, the initial frames are conveyed to the upper layer with
     // an invalid VideoDecoderConfig and it's up to the upper layer
     // to process this kind of frame accordingly.
     if (last_video_decoder_config_.IsValidConfig())
       return false;
@@ skipping 16 matching lines @@
   // type and allow multiple video tracks. See https://crbug.com/341581.
   scoped_refptr<StreamParserBuffer> stream_parser_buffer =
       StreamParserBuffer::CopyFrom(
           es,
           access_unit_size,
           is_key_frame,
           DemuxerStream::VIDEO,
           0);
   stream_parser_buffer->SetDecodeTimestamp(current_timing_desc.dts);
   stream_parser_buffer->set_timestamp(current_timing_desc.pts);
+  stream_parser_buffer->set_duration(frame_duration);
   emit_buffer_cb_.Run(stream_parser_buffer);
   return true;
 }
 
 bool EsParserH264::UpdateVideoDecoderConfig(const H264SPS* sps) {
   // Set the SAR to 1 when not specified in the H264 stream.
   int sar_width = (sps->sar_width == 0) ? 1 : sps->sar_width;
   int sar_height = (sps->sar_height == 0) ? 1 : sps->sar_height;
 
   // TODO(damienv): a MAP unit can be either 16 or 32 pixels.
@@ skipping 37 matching lines @@
     last_video_decoder_config_ = video_decoder_config;
     new_video_config_cb_.Run(video_decoder_config);
   }
 
   return true;
 }
 
 }  // namespace mp2t
 }  // namespace media