OLD | NEW |
(Empty) | |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #ifndef MEDIA_FILTERS_VIDEO_RENDERER_ALGORITHM_H_ |
| 6 #define MEDIA_FILTERS_VIDEO_RENDERER_ALGORITHM_H_ |
| 7 |
| 8 #include <deque> |
| 9 |
| 10 #include "base/callback.h" |
| 11 #include "base/memory/ref_counted.h" |
| 12 #include "base/time/time.h" |
| 13 #include "media/base/media_export.h" |
| 14 #include "media/base/moving_average.h" |
| 15 #include "media/base/video_frame.h" |
| 16 #include "media/base/video_renderer.h" |
| 17 #include "media/filters/video_cadence_estimator.h" |
| 18 |
| 19 namespace media { |
| 20 |
| 21 // VideoRendererAlgorithm manages a queue of VideoFrames from which it chooses |
| 22 // frames with the goal of providing a smooth playback experience. I.e., the |
| 23 // selection process results in the best possible uniformity for displayed frame |
| 24 // durations over time. |
| 25 // |
| 26 // Clients will provide frames to VRA via EnqueueFrame() and then VRA will yield |
| 27 // one of those frames in response to a future Render() call. Each Render() |
| 28 // call takes a render interval which is used to compute the best frame for |
| 29 // display during that interval. |
| 30 // |
| 31 // Render() calls are expected to happen on a regular basis. Failure to do so |
| 32 // will result in suboptimal rendering experiences. If a client knows that |
| 33 // Render() callbacks are stalled for any reason, it should tell VRA to expire |
| 34 // frames which are unusable via RemoveExpiredFrames(); this prevents useless |
| 35 // accumulation of stale VideoFrame objects (which are frequently quite large). |
| 36 // |
| 37 // The primary means of smooth frame selection is via forced integer cadence, |
| 38 // see VideoCadenceEstimator for details on this process. In cases of non- |
| 39 // integer cadence, the algorithm will fall back to choosing the frame which |
| 40 // covers the most of the current render interval. If no frame covers the |
| 41 // current interval, the least bad frame will be chosen based on its drift from |
| 42 // the start of the interval. |
| 43 // |
| 44 // Combined these three approaches enforce optimal smoothness in many cases. |
| 45 class MEDIA_EXPORT VideoRendererAlgorithm { |
| 46 public: |
| 47 // Used to convert a media timestamp into wall clock time. |
| 48 using TimeConverterCB = base::Callback<base::TimeTicks(base::TimeDelta)>; |
| 49 |
| 50 explicit VideoRendererAlgorithm(const TimeConverterCB& time_converter_cb); |
| 51 ~VideoRendererAlgorithm(); |
| 52 |
| 53 // Chooses the best frame for the interval [deadline_min, deadline_max] based |
| 54 // on available and previously rendered frames. |
| 55 // |
| 56 // Under ideal circumstances the deadline interval provided to a Render() call |
| 57 // should be directly adjacent to the deadline given to the previous Render() |
| 58 // call with no overlap or gaps. In practice, |deadline_max| is an estimated |
| 59 // value, which means the next |deadline_min| may overlap it slightly or have |
| 60 // a slight gap. Gaps which exceed the length of the deadline interval are |
| 61 // assumed to be repeated frames for the purposes of cadence detection. |
| 62 // |
| 63 // If provided, |frames_dropped| will be set to the number of frames which |
| 64 // were removed from |frame_queue_|, during this call, which were never |
| 65 // returned during a previous Render() call and are no longer suitable for |
| 66 // rendering since their wall clock time is too far in the past. |
| 67 scoped_refptr<VideoFrame> Render(base::TimeTicks deadline_min, |
| 68 base::TimeTicks deadline_max, |
| 69 size_t* frames_dropped); |
| 70 |
| 71 // Removes all video frames which are unusable since their ideal render |
| 72 // interval [timestamp, timestamp + duration] is too far away from |
| 73 // |deadline_min| than is allowed by drift constraints. |
| 74 // |
| 75 // At least one frame will always remain after this call so that subsequent |
| 76 // Render() calls have a frame to return if no new frames are enqueued before |
| 77 // then. Returns the number of frames removed. |
| 78 size_t RemoveExpiredFrames(base::TimeTicks deadline_min); |
| 79 |
| 80 // Clients should call this if the last frame provided by Render() was never |
| 81 // rendered; it ensures the presented cadence matches internal models. This |
| 82 // must be called before the next Render() call. |
| 83 void OnLastFrameDropped(); |
| 84 |
| 85 // Adds a frame to |frame_queue_| for consideration by Render(). Out of order |
| 86 // timestamps will be sorted into appropriate order. Do not enqueue end of |
| 87 // stream frames. Frames inserted prior to the last rendered frame will not |
| 88 // be used. They will be discarded on the next call to Render(), counting as |
| 89 // dropped frames, or by RemoveExpiredFrames(), counting as expired frames. |
| 90 void EnqueueFrame(const scoped_refptr<VideoFrame>& frame); |
| 91 |
| 92 // Removes all frames from the |frame_queue_| and clears predictors. The |
| 93 // algorithm will be as if freshly constructed after this call. |
| 94 void Reset(); |
| 95 |
| 96 // Returns the number of frames currently buffered which could be rendered |
| 97 // assuming current Render() interval trends. Before Render() is called or if |
| 98 // no cadence pattern is detected, this will be the same as the number of |
| 99 // frames given to EnqueueFrame(). |
| 100 // |
| 101 // If a cadence has been identified, this will return the number of frames |
| 102 // which have a non-zero ideal render count. |
| 103 size_t EffectiveFramesQueued() const; |
| 104 |
| 105 size_t frames_queued() const { return frame_queue_.size(); } |
| 106 |
| 107 // Returns the average of the duration of all frames in |frame_queue_| |
| 108 // as measured in wall clock (not media) time. |
| 109 base::TimeDelta average_frame_duration() const { |
| 110 return average_frame_duration_; |
| 111 } |
| 112 |
| 113 private: |
| 114 friend class VideoRendererAlgorithmTest; |
| 115 |
| 116 // The determination of whether to clamp to a given cadence is based on the |
| 117 // number of seconds before a frame would have to be dropped or repeated to |
| 118 // compensate for reaching the maximum acceptable drift. |
| 119 // |
| 120 // We've chosen 8 seconds based on practical observations and the fact that it |
| 121 // allows 29.9fps and 59.94fps in 60Hz and vice versa. |
| 122 // |
| 123 // Most users will not be able to see a single frame repeated or dropped every |
| 124 // 8 seconds and certainly should notice it less than the randomly variable |
| 125 // frame durations. |
| 126 static const int kMinimumAcceptableTimeBetweenGlitchesSecs = 8; |
| 127 |
| 128 // Metadata container for enqueued frames. See |frame_queue_| below. |
| 129 struct ReadyFrame { |
| 130 ReadyFrame(const scoped_refptr<VideoFrame>& frame); |
| 131 ~ReadyFrame(); |
| 132 |
| 133 // For use with std::lower_bound. |
| 134 bool operator<(const ReadyFrame& other) const; |
| 135 |
| 136 scoped_refptr<VideoFrame> frame; |
| 137 base::TimeTicks wall_clock_time; |
| 138 int ideal_render_count; |
| 139 int render_count; |
| 140 int drop_count; |
| 141 }; |
| 142 |
| 143 // Updates the render count for the last rendered frame based on the number |
| 144 // of missing intervals between Render() calls. |
| 145 void AccountForMissedIntervals(base::TimeTicks deadline_min, |
| 146 base::TimeTicks deadline_max); |
| 147 |
| 148 // Updates the render count and wall clock timestamps for all frames in |
| 149 // |frame_queue_|. Returns false if statistics can't be updated at this time; |
| 150 // which occurs if media time has stopped or there are not enough frames to |
| 151 // calculate an average frame duration. Updates |cadence_estimator_|. |
| 152 // |
| 153 // Note: Wall clock time is recomputed each Render() call because it's |
| 154 // expected that the TimeSource powering TimeConverterCB will skew slightly |
| 155 // based on the audio clock. |
| 156 // |
| 157 // TODO(dalecurtis): Investigate how accurate we need the wall clock times to |
| 158 // be, so we can avoid recomputing every time (we would need to recompute when |
| 159 // playback rate changes occur though). |
| 160 bool UpdateFrameStatistics(); |
| 161 |
| 162 // Updates the ideal render count for all frames in |frame_queue_| based on |
| 163 // the cadence returned by |cadence_estimator_|. |
| 164 void UpdateCadenceForFrames(); |
| 165 |
| 166 // If |cadence_estimator_| has detected a valid cadence, attempts to find the |
| 167 // next frame which should be rendered. Returns -1 if not enough frames are |
| 168 // available for cadence selection or there is no cadence. Will adjust the |
| 169 // selected frame's ideal render count if the last rendered frame has been |
| 170 // over selected. |
| 171 int FindBestFrameByCadence(); |
| 172 |
| 173 // Similar to FindBestFrameByCadence(), but instead of adjusting the last |
| 174 // rendered frame's ideal render count in the case of over selection, |
| 175 // optionally returns the new ideal render count via |
| 176 // |adjusted_ideal_render_count|. |
| 177 int FindBestFrameByCadenceInternal(int* adjusted_ideal_render_count) const; |
| 178 |
| 179 // Iterates over |frame_queue_| and finds the frame which covers the most of |
| 180 // the deadline interval. If multiple frames have coverage of the interval, |
| 181 // |second_best| will be set to the index of the frame with the next highest |
| 182 // coverage. Returns -1 if no frame has any coverage of the current interval. |
| 183 // |
| 184 // Prefers the earliest frame if multiple frames have similar coverage (within |
| 185 // a few percent of each other). |
| 186 int FindBestFrameByCoverage(base::TimeTicks deadline_min, |
| 187 base::TimeTicks deadline_max, |
| 188 int* second_best); |
| 189 |
| 190 // Iterates over |frame_queue_| and find the frame which drifts the least from |
| 191 // |deadline_min|. There's always a best frame by drift, so the return value |
| 192 // is always a valid frame index. |selected_frame_drift| will be set to the |
| 193 // drift of the chosen frame. |
| 194 int FindBestFrameByDrift(base::TimeTicks deadline_min, |
| 195 base::TimeDelta* selected_frame_drift); |
| 196 |
| 197 // Calculates the drift from |deadline_min| for the given |frame_index|. If |
| 198 // the [wall_clock_time, wall_clock_time + average_frame_duration_] lies |
| 199 // before |deadline_min| the drift is the delta between |deadline_min| and |
| 200 // |wall_clock_time + average_frame_duration_|. If the frame overlaps |
| 201 // |deadline_min| the drift is zero. If the frame lies after |deadline_min| |
| 202 // the drift is the delta between |deadline_min| and |wall_clock_time|. |
| 203 base::TimeDelta CalculateAbsoluteDriftForFrame(base::TimeTicks deadline_min, |
| 204 int frame_index); |
| 205 |
| 206 // Returns the wall clock time of the next frame if it exists, otherwise it |
| 207 // returns the time of the requested frame plus |average_frame_duration_|. |
| 208 base::TimeTicks EndTimeForFrame(size_t frame_index); |
| 209 |
| 210 // Queue of incoming frames waiting for rendering. |
| 211 using VideoFrameQueue = std::deque<ReadyFrame>; |
| 212 VideoFrameQueue frame_queue_; |
| 213 |
| 214 // The index of the last frame rendered; presumed to be the first frame if no |
| 215 // frame has been rendered yet. Updated by Render() and EnqueueFrame() if any |
| 216 // frames are added or removed. |
| 217 // |
| 218 // In most cases this value is zero, but when out of order timestamps are |
| 219 // present, the last rendered frame may be moved. |
| 220 size_t last_frame_index_; |
| 221 |
| 222 // Handles cadence detection and frame cadence assignments. |
| 223 VideoCadenceEstimator cadence_estimator_; |
| 224 |
| 225 // Indicates if any calls to Render() have successfully yielded a frame yet. |
| 226 bool have_rendered_frames_; |
| 227 |
| 228 // Callback used to convert media timestamps into wall clock timestamps. |
| 229 const TimeConverterCB time_converter_cb_; |
| 230 |
| 231 // The last |deadline_max| provided to Render(), used to predict whether |
| 232 // frames were rendered over cadence between Render() calls. |
| 233 base::TimeTicks last_deadline_max_; |
| 234 |
| 235 // The average of the duration of all frames in |frame_queue_| as measured in |
| 236 // wall clock (not media) time at the time of the last Render(). |
| 237 MovingAverage frame_duration_calculator_; |
| 238 base::TimeDelta average_frame_duration_; |
| 239 |
| 240 // The length of the last deadline interval given to Render(), updated at the |
| 241 // start of Render(). |
| 242 base::TimeDelta render_interval_; |
| 243 |
| 244 // The maximum acceptable drift before a frame can no longer be considered for |
| 245 // rendering within a given interval. |
| 246 base::TimeDelta max_acceptable_drift_; |
| 247 |
| 248 // Indicates that the last call to Render() experienced a rendering glitch; it |
| 249 // may have: under-rendered a frame, over-rendered a frame, dropped one or |
| 250 // more frames, or chosen a frame which exceeded acceptable drift. |
| 251 bool last_render_had_glitch_; |
| 252 |
| 253 DISALLOW_COPY_AND_ASSIGN(VideoRendererAlgorithm); |
| 254 }; |
| 255 |
| 256 } // namespace media |
| 257 |
| 258 #endif // MEDIA_FILTERS_VIDEO_RENDERER_ALGORITHM_H_ |
OLD | NEW |