OLD | NEW |
(Empty) | |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #ifndef MEDIA_FILTERS_VIDEO_RENDERER_ALGORITHM_H_ |
| 6 #define MEDIA_FILTERS_VIDEO_RENDERER_ALGORITHM_H_ |
| 7 |
| 8 #include <deque> |
| 9 |
| 10 #include "base/callback.h" |
| 11 #include "base/memory/ref_counted.h" |
| 12 #include "base/time/time.h" |
| 13 #include "media/base/media_export.h" |
| 14 #include "media/base/moving_average.h" |
| 15 #include "media/base/video_frame.h" |
| 16 #include "media/base/video_renderer.h" |
| 17 #include "media/filters/video_cadence_estimator.h" |
| 18 |
| 19 namespace media { |
| 20 |
| 21 // VideoRendererAlgorithm manages a queue of VideoFrames from which it chooses |
| 22 // frames with the goal of providing a smooth playback experience. I.e., the |
| 23 // selection process results in the best possible uniformity for displayed frame |
| 24 // durations over time. |
| 25 // |
| 26 // Clients will provide frames to VRA via EnqueueFrame() and then VRA will yield |
| 27 // one of those frames in response to a future Render() call. Each Render() |
| 28 // call takes a render interval which is used to compute the best frame for |
| 29 // display during that interval. |
| 30 // |
| 31 // Render() calls are expected to happen on a regular basis. Failure to do so |
| 32 // will result in suboptimal rendering experiences. If a client knows that |
| 33 // Render() callbacks are stalled for any reason, it should tell VRA to expire |
| 34 // frames which are unusable via RemoveExpiredFrames(); this prevents useless |
| 35 // accumulation of stale VideoFrame objects (which are frequently quite large). |
| 36 // |
| 37 // The primary means of smooth frame selection is via forced integer cadence, |
| 38 // see VideoCadenceEstimator for details on this process. In cases of non- |
| 39 // integer cadence, the algorithm will fall back to choosing the frame which |
| 40 // covers the most of the current render interval. If no frame covers the |
| 41 // current interval, the least bad frame will be chosen based on its drift from |
| 42 // the start of the interval. |
| 43 // |
| 44 // Combined these three approaches enforce optimal smoothness in many cases. |
| 45 class MEDIA_EXPORT VideoRendererAlgorithm { |
| 46 public: |
| 47 // Used to convert a media timestamp into wall clock time. |
| 48 using TimeConverterCB = base::Callback<base::TimeTicks(base::TimeDelta)>; |
| 49 |
| 50 explicit VideoRendererAlgorithm(const TimeConverterCB& time_converter_cb); |
| 51 ~VideoRendererAlgorithm(); |
| 52 |
| 53 // Chooses the best frame for the interval [deadline_min, deadline_max] based |
| 54 // on available and previously rendered frames. |
| 55 // |
| 56 // Under ideal circumstances the deadline interval provided to a Render() call |
| 57 // should be directly adjacent to the deadline given to the previous Render() |
| 58 // call with no overlap or gaps. In practice, |deadline_max| is an estimated |
| 59 // value, which means the next |deadline_min| may overlap it slightly or have |
| 60 // a slight gap. Gaps which exceed the length of the deadline interval are |
| 61 // assumed to be repeated frames for the purposes of cadence detection. |
| 62 // |
| 63 // If provided, |frames_dropped| will be set to the number of frames which |
| 64 // were removed from |frame_queue_|, during this call, which were never |
| 65 // returned during a previous Render() call and are no longer suitable for |
| 66 // rendering since their wall clock time is too far in the past. |
| 67 scoped_refptr<VideoFrame> Render(base::TimeTicks deadline_min, |
| 68 base::TimeTicks deadline_max, |
| 69 size_t* frames_dropped); |
| 70 |
| 71 // Removes all video frames which are unusable since their ideal render |
| 72 // interval [timestamp, timestamp + duration] is too far away from |
| 73 // |deadline_min| than is allowed by drift constraints. |
| 74 // |
| 75 // At least one frame will always remain after this call so that subsequent |
| 76 // Render() calls have a frame to return if no new frames are enqueued before |
| 77 // then. Returns the number of frames removed. |
| 78 // |
| 79 // Note: In cases where there is no known frame duration (i.e. perhaps a video |
| 80 // with only a single frame), the last frame can not be expired, regardless of |
| 81 // the given deadline. Clients must handle this case externally. |
| 82 size_t RemoveExpiredFrames(base::TimeTicks deadline); |
| 83 |
| 84 // Clients should call this if the last frame provided by Render() was never |
| 85 // rendered; it ensures the presented cadence matches internal models. This |
| 86 // must be called before the next Render() call. |
| 87 void OnLastFrameDropped(); |
| 88 |
| 89 // Adds a frame to |frame_queue_| for consideration by Render(). Out of order |
| 90 // timestamps will be sorted into appropriate order. Do not enqueue end of |
| 91 // stream frames. Frames inserted prior to the last rendered frame will not |
| 92 // be used. They will be discarded on the next call to Render(), counting as |
| 93 // dropped frames, or by RemoveExpiredFrames(), counting as expired frames. |
| 94 void EnqueueFrame(const scoped_refptr<VideoFrame>& frame); |
| 95 |
| 96 // Removes all frames from the |frame_queue_| and clears predictors. The |
| 97 // algorithm will be as if freshly constructed after this call. |
| 98 void Reset(); |
| 99 |
| 100 // Returns the number of frames currently buffered which could be rendered |
| 101 // assuming current Render() interval trends. Before Render() is called, this |
| 102 // will be the same as the number of frames given to EnqueueFrame(). After |
| 103 // Render() has been called, one of two things will be returned: |
| 104 // |
| 105 // If a cadence has been identified, this will return the number of frames |
| 106 // which have a non-zero ideal render count. |
| 107 // |
| 108 // If cadence has not been identified, this will return the number of frames |
| 109 // which have a frame end time greater than the end of the last render |
| 110 // interval passed to Render(). Note: If Render() callbacks become suspended |
| 111 // and the duration is unknown the last frame may never be stop counting as |
| 112 // effective. Clients must handle this case externally. |
| 113 // |
| 114 // In either case, frames enqueued before the last displayed frame will not |
| 115 // be counted as effective. |
| 116 size_t EffectiveFramesQueued() const; |
| 117 |
| 118 size_t frames_queued() const { return frame_queue_.size(); } |
| 119 |
| 120 // Returns the average of the duration of all frames in |frame_queue_| |
| 121 // as measured in wall clock (not media) time. |
| 122 base::TimeDelta average_frame_duration() const { |
| 123 return average_frame_duration_; |
| 124 } |
| 125 |
| 126 // Method used for testing which disables frame dropping, in this mode the |
| 127 // algorithm will never drop frames and instead always return every frame |
| 128 // for display at least once. |
| 129 void disable_frame_dropping() { frame_dropping_disabled_ = true; } |
| 130 |
| 131 private: |
| 132 friend class VideoRendererAlgorithmTest; |
| 133 |
| 134 // The determination of whether to clamp to a given cadence is based on the |
| 135 // number of seconds before a frame would have to be dropped or repeated to |
| 136 // compensate for reaching the maximum acceptable drift. |
| 137 // |
| 138 // We've chosen 8 seconds based on practical observations and the fact that it |
| 139 // allows 29.9fps and 59.94fps in 60Hz and vice versa. |
| 140 // |
| 141 // Most users will not be able to see a single frame repeated or dropped every |
| 142 // 8 seconds and certainly should notice it less than the randomly variable |
| 143 // frame durations. |
| 144 static const int kMinimumAcceptableTimeBetweenGlitchesSecs = 8; |
| 145 |
| 146 // Metadata container for enqueued frames. See |frame_queue_| below. |
| 147 struct ReadyFrame { |
| 148 ReadyFrame(const scoped_refptr<VideoFrame>& frame); |
| 149 ~ReadyFrame(); |
| 150 |
| 151 // For use with std::lower_bound. |
| 152 bool operator<(const ReadyFrame& other) const; |
| 153 |
| 154 scoped_refptr<VideoFrame> frame; |
| 155 base::TimeTicks wall_clock_time; |
| 156 int ideal_render_count; |
| 157 int render_count; |
| 158 int drop_count; |
| 159 }; |
| 160 |
| 161 // Updates the render count for the last rendered frame based on the number |
| 162 // of missing intervals between Render() calls. |
| 163 void AccountForMissedIntervals(base::TimeTicks deadline_min, |
| 164 base::TimeTicks deadline_max); |
| 165 |
| 166 // Updates the render count and wall clock timestamps for all frames in |
| 167 // |frame_queue_|. Returns false if statistics can't be updated at this time; |
| 168 // which occurs if media time has stopped or there are not enough frames to |
| 169 // calculate an average frame duration. Updates |cadence_estimator_|. |
| 170 // |
| 171 // Note: Wall clock time is recomputed each Render() call because it's |
| 172 // expected that the TimeSource powering TimeConverterCB will skew slightly |
| 173 // based on the audio clock. |
| 174 // |
| 175 // TODO(dalecurtis): Investigate how accurate we need the wall clock times to |
| 176 // be, so we can avoid recomputing every time (we would need to recompute when |
| 177 // playback rate changes occur though). |
| 178 bool UpdateFrameStatistics(); |
| 179 |
| 180 // Updates the ideal render count for all frames in |frame_queue_| based on |
| 181 // the cadence returned by |cadence_estimator_|. |
| 182 void UpdateCadenceForFrames(); |
| 183 |
| 184 // If |cadence_estimator_| has detected a valid cadence, attempts to find the |
| 185 // next frame which should be rendered. Returns -1 if not enough frames are |
| 186 // available for cadence selection or there is no cadence. Will adjust the |
| 187 // selected frame's ideal render count if the last rendered frame has been |
| 188 // over selected. |
| 189 int FindBestFrameByCadence(); |
| 190 |
| 191 // Similar to FindBestFrameByCadence(), but instead of adjusting the last |
| 192 // rendered frame's ideal render count in the case of over selection, |
| 193 // optionally returns the new ideal render count via |
| 194 // |adjusted_ideal_render_count|. |
| 195 int FindBestFrameByCadenceInternal(int* adjusted_ideal_render_count) const; |
| 196 |
| 197 // Iterates over |frame_queue_| and finds the frame which covers the most of |
| 198 // the deadline interval. If multiple frames have coverage of the interval, |
| 199 // |second_best| will be set to the index of the frame with the next highest |
| 200 // coverage. Returns -1 if no frame has any coverage of the current interval. |
| 201 // |
| 202 // Prefers the earliest frame if multiple frames have similar coverage (within |
| 203 // a few percent of each other). |
| 204 int FindBestFrameByCoverage(base::TimeTicks deadline_min, |
| 205 base::TimeTicks deadline_max, |
| 206 int* second_best) const; |
| 207 |
| 208 // Iterates over |frame_queue_| and find the frame which drifts the least from |
| 209 // |deadline_min|. There's always a best frame by drift, so the return value |
| 210 // is always a valid frame index. |selected_frame_drift| will be set to the |
| 211 // drift of the chosen frame. |
| 212 // |
| 213 // Note: Drift calculations assume contiguous frames in the time domain, so |
| 214 // it's not possible to have a case where a frame is -10ms from |deadline_min| |
| 215 // and another frame which is at some time after |deadline_min|. The second |
| 216 // frame would be considered to start at -10ms before |deadline_min| and would |
| 217 // overlap |deadline_min|, so its drift would be zero. |
| 218 int FindBestFrameByDrift(base::TimeTicks deadline_min, |
| 219 base::TimeDelta* selected_frame_drift) const; |
| 220 |
| 221 // Calculates the drift from |deadline_min| for the given |frame_index|. If |
| 222 // the [wall_clock_time, wall_clock_time + average_frame_duration_] lies |
| 223 // before |deadline_min| the drift is the delta between |deadline_min| and |
| 224 // |wall_clock_time + average_frame_duration_|. If the frame overlaps |
| 225 // |deadline_min| the drift is zero. If the frame lies after |deadline_min| |
| 226 // the drift is the delta between |deadline_min| and |wall_clock_time|. |
| 227 base::TimeDelta CalculateAbsoluteDriftForFrame(base::TimeTicks deadline_min, |
| 228 int frame_index) const; |
| 229 |
| 230 // Returns the wall clock time of the next frame if it exists, otherwise it |
| 231 // returns the time of the requested frame plus |average_frame_duration_|. |
| 232 base::TimeTicks EndTimeForFrame(size_t frame_index) const; |
| 233 |
| 234 // Queue of incoming frames waiting for rendering. |
| 235 using VideoFrameQueue = std::deque<ReadyFrame>; |
| 236 VideoFrameQueue frame_queue_; |
| 237 |
| 238 // The index of the last frame rendered; presumed to be the first frame if no |
| 239 // frame has been rendered yet. Updated by Render() and EnqueueFrame() if any |
| 240 // frames are added or removed. |
| 241 // |
| 242 // In most cases this value is zero, but when out of order timestamps are |
| 243 // present, the last rendered frame may be moved. |
| 244 size_t last_frame_index_; |
| 245 |
| 246 // Handles cadence detection and frame cadence assignments. |
| 247 VideoCadenceEstimator cadence_estimator_; |
| 248 |
| 249 // Indicates if any calls to Render() have successfully yielded a frame yet. |
| 250 bool have_rendered_frames_; |
| 251 |
| 252 // Callback used to convert media timestamps into wall clock timestamps. |
| 253 const TimeConverterCB time_converter_cb_; |
| 254 |
| 255 // The last |deadline_max| provided to Render(), used to predict whether |
| 256 // frames were rendered over cadence between Render() calls. |
| 257 base::TimeTicks last_deadline_max_; |
| 258 |
| 259 // The average of the duration of all frames in |frame_queue_| as measured in |
| 260 // wall clock (not media) time at the time of the last Render(). |
| 261 MovingAverage frame_duration_calculator_; |
| 262 base::TimeDelta average_frame_duration_; |
| 263 |
| 264 // The length of the last deadline interval given to Render(), updated at the |
| 265 // start of Render(). |
| 266 base::TimeDelta render_interval_; |
| 267 |
| 268 // The maximum acceptable drift before a frame can no longer be considered for |
| 269 // rendering within a given interval. |
| 270 base::TimeDelta max_acceptable_drift_; |
| 271 |
| 272 // Indicates that the last call to Render() experienced a rendering glitch; it |
| 273 // may have: under-rendered a frame, over-rendered a frame, dropped one or |
| 274 // more frames, or chosen a frame which exceeded acceptable drift. |
| 275 bool last_render_had_glitch_; |
| 276 |
| 277 // For testing functionality which enables clockless playback of all frames. |
| 278 bool frame_dropping_disabled_; |
| 279 |
| 280 DISALLOW_COPY_AND_ASSIGN(VideoRendererAlgorithm); |
| 281 }; |
| 282 |
| 283 } // namespace media |
| 284 |
| 285 #endif // MEDIA_FILTERS_VIDEO_RENDERER_ALGORITHM_H_ |
OLD | NEW |