Improving WebM video duration estimation.

media/formats/webm/webm_cluster_parser.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/formats/webm/webm_cluster_parser.h"
 
 #include <vector>
 
 #include "base/logging.h"
 #include "base/sys_byteorder.h"
 #include "media/base/buffers.h"
 #include "media/base/decrypt_config.h"
 #include "media/filters/webvtt_util.h"
 #include "media/formats/webm/webm_constants.h"
 #include "media/formats/webm/webm_crypto_helpers.h"
 #include "media/formats/webm/webm_webvtt_parser.h"
 
 namespace media {
 
 const uint16_t WebMClusterParser::kOpusFrameDurationsMu[] = {
     10000, 20000, 40000, 60000, 10000, 20000, 40000, 60000, 10000, 20000, 40000,
     60000, 10000, 20000, 10000, 20000, 2500,  5000,  10000, 20000, 2500,  5000,
     10000, 20000, 2500,  5000,  10000, 20000, 2500,  5000,  10000, 20000};
 
 enum {
   // Limits the number of MEDIA_LOG() calls in the path of reading encoded
   // duration to avoid spamming for corrupted data.
-  kMaxDurationLogs = 10,
+  kMaxDurationErrorLogs = 10,
+  // Limits the number of MEDIA_LOG() calls warning the user that a buffer
+  // durations have been estimated.

[wolenetz, 2015/03/28 00:26:06]

nit: regrammarize "a buffer durations have" ;)

[chcunningham, 2015/04/13 23:25:18]

Done.

+  kMaxDurationEstimateLogs = 10,
 };
 
 WebMClusterParser::WebMClusterParser(
     int64 timecode_scale,
     int audio_track_num,
     base::TimeDelta audio_default_duration,
     int video_track_num,
     base::TimeDelta video_default_duration,
     const WebMTracksParser::TextTracks& text_tracks,
     const std::set<int64>& ignored_tracks,
@@ skipping 139 matching lines @@
                                                     int size) {
   // Masks and constants for Opus packets. See
   // https://tools.ietf.org/html/rfc6716#page-14
   static const uint8_t kTocConfigMask = 0xf8;
   static const uint8_t kTocFrameCountCodeMask = 0x03;
   static const uint8_t kFrameCountMask = 0x3f;
   static const base::TimeDelta kPacketDurationMax =
       base::TimeDelta::FromMilliseconds(120);
 
   if (size < 1) {
-    LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationLogs)
+    LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationErrorLogs)
         << "Invalid zero-byte Opus packet; demuxed block duration may be "
            "imprecise.";
     return kNoTimestamp();
   }
 
   // Frame count type described by last 2 bits of Opus TOC byte.
   int frame_count_type = data[0] & kTocFrameCountCodeMask;
 
   int frame_count = 0;
   switch (frame_count_type) {
     case 0:
       frame_count = 1;
       break;
     case 1:
     case 2:
       frame_count = 2;
       break;
     case 3:
       // Type 3 indicates an arbitrary frame count described in the next byte.
       if (size < 2) {
-        LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationLogs)
+        LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationErrorLogs)
             << "Second byte missing from 'Code 3' Opus packet; demuxed block "
                "duration may be imprecise.";
         return kNoTimestamp();
       }
 
       frame_count = data[1] & kFrameCountMask;
 
       if (frame_count == 0) {
-        LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationLogs)
+        LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationErrorLogs)
             << "Illegal 'Code 3' Opus packet with frame count zero; demuxed "
                "block duration may be imprecise.";
         return kNoTimestamp();
       }
 
       break;
     default:
-      LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationLogs)
+      LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationErrorLogs)
           << "Unexpected Opus frame count type: " << frame_count_type << "; "
           << "demuxed block duration may be imprecise.";
       return kNoTimestamp();
   }
 
   int opusConfig = (data[0] & kTocConfigMask) >> 3;
   CHECK_GE(opusConfig, 0);
   CHECK_LT(opusConfig, static_cast<int>(arraysize(kOpusFrameDurationsMu)));
 
   DCHECK_GT(frame_count, 0);
   base::TimeDelta duration = base::TimeDelta::FromMicroseconds(
       kOpusFrameDurationsMu[opusConfig] * frame_count);
 
   if (duration > kPacketDurationMax) {
     // Intentionally allowing packet to pass through for now. Decoder should
     // either handle or fail gracefully. MEDIA_LOG as breadcrumbs in case
     // things go sideways.
-    LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationLogs)
+    LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationErrorLogs)
         << "Warning, demuxed Opus packet with encoded duration: " << duration
         << ". Should be no greater than " << kPacketDurationMax;
   }
 
   return duration;
 }
 
 WebMParserClient* WebMClusterParser::OnListStart(int id) {
   if (id == kWebMIdCluster) {
     cluster_timecode_ = -1;
@@ skipping 296 matching lines @@
     DVLOG(3) << __FUNCTION__ << " : "
              << "Using encoded duration " << encoded_duration.InSecondsF();
 
     if (block_duration_time_delta != kNoTimestamp()) {
       base::TimeDelta duration_difference =
           block_duration_time_delta - encoded_duration;
 
       const auto kWarnDurationDiff =
           base::TimeDelta::FromMicroseconds(timecode_multiplier_ * 2);
       if (duration_difference.magnitude() > kWarnDurationDiff) {
-        LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationLogs)
+        LIMITED_MEDIA_LOG(log_cb_, num_duration_errors_, kMaxDurationErrorLogs)
             << "BlockDuration "
             << "(" << block_duration_time_delta << ") "
             << "differs significantly from encoded duration "
             << "(" << encoded_duration << ").";
       }
     }
   } else if (block_duration_time_delta != kNoTimestamp()) {
     buffer->set_duration(block_duration_time_delta);
   } else {
     DCHECK_NE(buffer_type, DemuxerStream::TEXT);
     buffer->set_duration(track->default_duration());
   }
 
   if (discard_padding != 0) {
     buffer->set_discard_padding(std::make_pair(
         base::TimeDelta(),
         base::TimeDelta::FromMicroseconds(discard_padding / 1000)));
   }
 
   return track->AddBuffer(buffer);
 }
 
 WebMClusterParser::Track::Track(int track_num,
                                 bool is_video,
                                 base::TimeDelta default_duration,
                                 const LogCB& log_cb)
-    : track_num_(track_num),
+    : num_duration_estimates_(0),
+      track_num_(track_num),
       is_video_(is_video),
       default_duration_(default_duration),
       estimated_next_frame_duration_(kNoTimestamp()),
       log_cb_(log_cb) {
   DCHECK(default_duration_ == kNoTimestamp() ||
          default_duration_ > base::TimeDelta());
 }
 
 WebMClusterParser::Track::~Track() {}
 
@@ skipping 72 matching lines @@
     return true;
   }
 
   return QueueBuffer(buffer);
 }
 
 void WebMClusterParser::Track::ApplyDurationEstimateIfNeeded() {
   if (!last_added_buffer_missing_duration_.get())
     return;
 
-  last_added_buffer_missing_duration_->set_duration(GetDurationEstimate());
+  base::TimeDelta estimated_duration = GetDurationEstimate();
+  last_added_buffer_missing_duration_->set_duration(estimated_duration);
+
+  // Exposing estimation so splicing/overlap frame processing can make informed
+  // decisions downstream.
+  last_added_buffer_missing_duration_->set_is_duration_estimated(true);
+
+  LIMITED_MEDIA_LOG(log_cb_, num_duration_estimates_, kMaxDurationEstimateLogs)
+      << "Estimating duration to be " << estimated_duration << " for the "
+      << "last (Simple)Block in the Cluster. Use BlockDuration at the end of "

[wolenetz, 2015/03/28 00:26:06]

nit: "in the Cluster for this Track". And "Instead of SimpleBlocks, use
BlockGroups with BlockDurations at the end of each Track in a Cluster to avoid
estimation."

[chcunningham, 2015/04/13 23:25:18]

Done.

+      << "Clusters to avoid estimation.";
 
   DVLOG(2) << "ApplyDurationEstimateIfNeeded() : new dur : "
            << " ts "
            << last_added_buffer_missing_duration_->timestamp().InSecondsF()
            << " dur "
            << last_added_buffer_missing_duration_->duration().InSecondsF()
            << " kf " << last_added_buffer_missing_duration_->is_key_frame()
            << " size " << last_added_buffer_missing_duration_->data_size();
 
   // Don't use the applied duration as a future estimation (don't use
@@ skipping 47 matching lines @@
   DecodeTimestamp previous_buffers_timestamp = buffers_.empty() ?
       DecodeTimestamp() : buffers_.back()->GetDecodeTimestamp();
   CHECK(previous_buffers_timestamp <= buffer->GetDecodeTimestamp());
 
   base::TimeDelta duration = buffer->duration();
   if (duration < base::TimeDelta() || duration == kNoTimestamp()) {
     MEDIA_LOG(log_cb_) << "Invalid buffer duration: " << duration.InSecondsF();
     return false;
   }
 
-  // The estimated frame duration is the minimum non-zero duration since the
-  // last initialization segment.  The minimum is used to ensure frame durations
-  // aren't overestimated.
+  // The estimated frame duration is the minimum (for audio) or the maximum
+  // (for video) non-zero duration since the last initialization segment. The
+  // minimum is used for audio to ensure frame durations aren't overestimated,
+  // triggering unnecessary frame splicing. For video, splicing does not apply,
+  // so maximum is used and overlap is is simply resolved by showing the

[wolenetz, 2015/03/28 00:26:06]

nit: is is

[chcunningham, 2015/04/13 23:25:18]

Done.

+  // later of the overlapping frames at its given PTS, effectively trimming down
+  // the over-estimated duration of the previous frame.
+  // TODO(chcunningham): Use max for audio and disable splicing whenever
+  // estimated buffers are encountered.
   if (duration > base::TimeDelta()) {
+    base::TimeDelta orig_duration_estimate = estimated_next_frame_duration_;
     if (estimated_next_frame_duration_ == kNoTimestamp()) {
       estimated_next_frame_duration_ = duration;
+    } else if (is_video_) {
+      estimated_next_frame_duration_ =
+          std::max(duration, estimated_next_frame_duration_);
     } else {
       estimated_next_frame_duration_ =
           std::min(duration, estimated_next_frame_duration_);
     }
+
+    if (orig_duration_estimate != estimated_next_frame_duration_) {
+      DVLOG(3) << "Updated duration estimate:"
+               << orig_duration_estimate
+               << " -> "
+               << estimated_next_frame_duration_
+               << " at dts: "

[wolenetz, 2015/03/28 00:26:06]

nit: webm doesn't differentiate dts/pts. s/dts/timestamp/
aside: it's weird that even inside webm parser we sometimes use
buffer->timestamp() and other times use buffer->GetDecodeTimestamp(). Seems ripe
for some low-pri cleanup.

[chcunningham, 2015/04/13 23:25:18]

Done. Re-cleanup: so you'd like a bug to change all DecodeTimestamp -> Timestamp
? I don't mind the using DecodeTimestamp, since thats intuitively whats most
important to sequencing buffers in a parser and the fact that WebM doesn't
differentiate is not obvious. If it were me, I'd always use DecodeTimestmp

[wolenetz, 2015/04/15 02:55:23]

Thanks. IIRC, WebM container doesn't mention PTS or DTS, just timestamp.
Internally here, it would be better to be consistent in usage of DTS or PTS on
the |buffer| accessor, but that's for a really low pri cleanup :)

+               << buffer->GetDecodeTimestamp().InSecondsF();
+    }
   }
 
   buffers_.push_back(buffer);
   return true;
 }
 
 base::TimeDelta WebMClusterParser::Track::GetDurationEstimate() {
   base::TimeDelta duration = estimated_next_frame_duration_;
   if (duration != kNoTimestamp()) {
     DVLOG(3) << __FUNCTION__ << " : using estimated duration";
@@ skipping 66 matching lines @@
 WebMClusterParser::FindTextTrack(int track_num) {
   const TextTrackMap::iterator it = text_track_map_.find(track_num);
 
   if (it == text_track_map_.end())
     return NULL;
 
   return &it->second;
 }
 
 }  // namespace media