| OLD | NEW |
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "content/browser/media/capture/video_capture_oracle.h" | 5 #include "content/browser/media/capture/animated_content_sampler.h" |
| 6 | 6 |
| 7 #include <algorithm> | 7 #include <algorithm> |
| 8 | 8 |
| 9 #include "base/format_macros.h" | |
| 10 #include "base/strings/stringprintf.h" | |
| 11 #include "base/trace_event/trace_event.h" | |
| 12 | |
| 13 namespace content { | 9 namespace content { |
| 14 | 10 |
| 15 namespace { | 11 namespace { |
| 16 | 12 |
| 17 // This value controls how many redundant, timer-base captures occur when the | |
| 18 // content is static. Redundantly capturing the same frame allows iterative | |
| 19 // quality enhancement, and also allows the buffer to fill in "buffered mode". | |
| 20 // | |
| 21 // TODO(nick): Controlling this here is a hack and a layering violation, since | |
| 22 // it's a strategy specific to the WebRTC consumer, and probably just papers | |
| 23 // over some frame dropping and quality bugs. It should either be controlled at | |
| 24 // a higher level, or else redundant frame generation should be pushed down | |
| 25 // further into the WebRTC encoding stack. | |
| 26 const int kNumRedundantCapturesOfStaticContent = 200; | |
| 27 | |
| 28 // These specify the minimum/maximum amount of recent event history to examine | 13 // These specify the minimum/maximum amount of recent event history to examine |
| 29 // to detect animated content. If the values are too low, there is a greater | 14 // to detect animated content. If the values are too low, there is a greater |
| 30 // risk of false-positive detections and low accuracy. If they are too high, | 15 // risk of false-positive detections and low accuracy. If they are too high, |
| 31 // the the implementation will be slow to lock-in/out, and also will not react | 16 // the the implementation will be slow to lock-in/out, and also will not react |
| 32 // well to mildly-variable frame rate content (e.g., 25 +/- 1 FPS). | 17 // well to mildly-variable frame rate content (e.g., 25 +/- 1 FPS). |
| 33 // | 18 // |
| 34 // These values were established by experimenting with a wide variety of | 19 // These values were established by experimenting with a wide variety of |
| 35 // scenarios, including 24/25/30 FPS videos, 60 FPS WebGL demos, and the | 20 // scenarios, including 24/25/30 FPS videos, 60 FPS WebGL demos, and the |
| 36 // transitions between static and animated content. | 21 // transitions between static and animated content. |
| 37 const int kMinObservationWindowMillis = 1000; | 22 const int kMinObservationWindowMillis = 1000; |
| 38 const int kMaxObservationWindowMillis = 2000; | 23 const int kMaxObservationWindowMillis = 2000; |
| 39 | 24 |
| 40 // The maximum amount of time that can elapse before declaring two subsequent | 25 // The maximum amount of time that can elapse before declaring two subsequent |
| 41 // events as "not animating." This is the same value found in | 26 // events as "not animating." This is the same value found in |
| 42 // cc::FrameRateCounter. | 27 // cc::FrameRateCounter. |
| 43 const int kNonAnimatingThresholdMillis = 250; // 4 FPS | 28 const int kNonAnimatingThresholdMillis = 250; // 4 FPS |
| 44 | 29 |
| 45 // The slowest that content can be animating in order for AnimatedContentSampler | 30 // The slowest that content can be animating in order for AnimatedContentSampler |
| 46 // to lock-in. This is the threshold at which the "smoothness" problem is no | 31 // to lock-in. This is the threshold at which the "smoothness" problem is no |
| 47 // longer relevant. | 32 // longer relevant. |
| 48 const int kMaxLockInPeriodMicros = 83333; // 12 FPS | 33 const int kMaxLockInPeriodMicros = 83333; // 12 FPS |
| 49 | 34 |
| 50 // The amount of time over which to fully correct the drift of the rewritten | 35 // The amount of time over which to fully correct the drift of the rewritten |
| 51 // frame timestamps from the presentation event timestamps. The lower the | 36 // frame timestamps from the presentation event timestamps. The lower the |
| 52 // value, the higher the variance in frame timestamps. | 37 // value, the higher the variance in frame timestamps. |
| 53 const int kDriftCorrectionMillis = 2000; | 38 const int kDriftCorrectionMillis = 2000; |
| 54 | 39 |
| 55 // Given the amount of time between frames, compare to the expected amount of | |
| 56 // time between frames at |frame_rate| and return the fractional difference. | |
| 57 double FractionFromExpectedFrameRate(base::TimeDelta delta, int frame_rate) { | |
| 58 DCHECK_GT(frame_rate, 0); | |
| 59 const base::TimeDelta expected_delta = | |
| 60 base::TimeDelta::FromSeconds(1) / frame_rate; | |
| 61 return (delta - expected_delta).InMillisecondsF() / | |
| 62 expected_delta.InMillisecondsF(); | |
| 63 } | |
| 64 | |
| 65 } // anonymous namespace | 40 } // anonymous namespace |
| 66 | 41 |
| 67 VideoCaptureOracle::VideoCaptureOracle(base::TimeDelta min_capture_period) | |
| 68 : frame_number_(0), | |
| 69 last_delivered_frame_number_(-1), | |
| 70 smoothing_sampler_(min_capture_period, | |
| 71 kNumRedundantCapturesOfStaticContent), | |
| 72 content_sampler_(min_capture_period) { | |
| 73 } | |
| 74 | |
| 75 VideoCaptureOracle::~VideoCaptureOracle() {} | |
| 76 | |
| 77 bool VideoCaptureOracle::ObserveEventAndDecideCapture( | |
| 78 Event event, | |
| 79 const gfx::Rect& damage_rect, | |
| 80 base::TimeTicks event_time) { | |
| 81 DCHECK_GE(event, 0); | |
| 82 DCHECK_LT(event, kNumEvents); | |
| 83 if (event_time < last_event_time_[event]) { | |
| 84 LOG(WARNING) << "Event time is not monotonically non-decreasing. " | |
| 85 << "Deciding not to capture this frame."; | |
| 86 return false; | |
| 87 } | |
| 88 last_event_time_[event] = event_time; | |
| 89 | |
| 90 bool should_sample; | |
| 91 switch (event) { | |
| 92 case kCompositorUpdate: | |
| 93 smoothing_sampler_.ConsiderPresentationEvent(event_time); | |
| 94 content_sampler_.ConsiderPresentationEvent(damage_rect, event_time); | |
| 95 if (content_sampler_.HasProposal()) { | |
| 96 should_sample = content_sampler_.ShouldSample(); | |
| 97 if (should_sample) | |
| 98 event_time = content_sampler_.frame_timestamp(); | |
| 99 } else { | |
| 100 should_sample = smoothing_sampler_.ShouldSample(); | |
| 101 } | |
| 102 break; | |
| 103 default: | |
| 104 should_sample = smoothing_sampler_.IsOverdueForSamplingAt(event_time); | |
| 105 break; | |
| 106 } | |
| 107 | |
| 108 SetFrameTimestamp(frame_number_, event_time); | |
| 109 return should_sample; | |
| 110 } | |
| 111 | |
| 112 int VideoCaptureOracle::RecordCapture() { | |
| 113 smoothing_sampler_.RecordSample(); | |
| 114 content_sampler_.RecordSample(GetFrameTimestamp(frame_number_)); | |
| 115 return frame_number_++; | |
| 116 } | |
| 117 | |
| 118 bool VideoCaptureOracle::CompleteCapture(int frame_number, | |
| 119 base::TimeTicks* frame_timestamp) { | |
| 120 // Drop frame if previous frame number is higher. | |
| 121 if (last_delivered_frame_number_ > frame_number) { | |
| 122 LOG(WARNING) << "Out of order frame delivery detected (have #" | |
| 123 << frame_number << ", last was #" | |
| 124 << last_delivered_frame_number_ << "). Dropping frame."; | |
| 125 return false; | |
| 126 } | |
| 127 last_delivered_frame_number_ = frame_number; | |
| 128 | |
| 129 *frame_timestamp = GetFrameTimestamp(frame_number); | |
| 130 | |
| 131 // If enabled, log a measurement of how this frame timestamp has incremented | |
| 132 // in relation to an ideal increment. | |
| 133 if (VLOG_IS_ON(2) && frame_number > 0) { | |
| 134 const base::TimeDelta delta = | |
| 135 *frame_timestamp - GetFrameTimestamp(frame_number - 1); | |
| 136 if (content_sampler_.HasProposal()) { | |
| 137 const double estimated_frame_rate = | |
| 138 1000000.0 / content_sampler_.detected_period().InMicroseconds(); | |
| 139 const int rounded_frame_rate = | |
| 140 static_cast<int>(estimated_frame_rate + 0.5); | |
| 141 VLOG(2) << base::StringPrintf( | |
| 142 "Captured #%d: delta=%" PRId64 " usec" | |
| 143 ", now locked into {%s}, %+0.1f%% slower than %d FPS", | |
| 144 frame_number, | |
| 145 delta.InMicroseconds(), | |
| 146 content_sampler_.detected_region().ToString().c_str(), | |
| 147 100.0 * FractionFromExpectedFrameRate(delta, rounded_frame_rate), | |
| 148 rounded_frame_rate); | |
| 149 } else { | |
| 150 VLOG(2) << base::StringPrintf( | |
| 151 "Captured #%d: delta=%" PRId64 " usec" | |
| 152 ", d/30fps=%+0.1f%%, d/25fps=%+0.1f%%, d/24fps=%+0.1f%%", | |
| 153 frame_number, | |
| 154 delta.InMicroseconds(), | |
| 155 100.0 * FractionFromExpectedFrameRate(delta, 30), | |
| 156 100.0 * FractionFromExpectedFrameRate(delta, 25), | |
| 157 100.0 * FractionFromExpectedFrameRate(delta, 24)); | |
| 158 } | |
| 159 } | |
| 160 | |
| 161 return !frame_timestamp->is_null(); | |
| 162 } | |
| 163 | |
| 164 base::TimeTicks VideoCaptureOracle::GetFrameTimestamp(int frame_number) const { | |
| 165 DCHECK_LE(frame_number, frame_number_); | |
| 166 DCHECK_LT(frame_number_ - frame_number, kMaxFrameTimestamps); | |
| 167 return frame_timestamps_[frame_number % kMaxFrameTimestamps]; | |
| 168 } | |
| 169 | |
| 170 void VideoCaptureOracle::SetFrameTimestamp(int frame_number, | |
| 171 base::TimeTicks timestamp) { | |
| 172 frame_timestamps_[frame_number % kMaxFrameTimestamps] = timestamp; | |
| 173 } | |
| 174 | |
| 175 SmoothEventSampler::SmoothEventSampler(base::TimeDelta min_capture_period, | |
| 176 int redundant_capture_goal) | |
| 177 : min_capture_period_(min_capture_period), | |
| 178 redundant_capture_goal_(redundant_capture_goal), | |
| 179 token_bucket_capacity_(min_capture_period + min_capture_period / 2), | |
| 180 overdue_sample_count_(0), | |
| 181 token_bucket_(token_bucket_capacity_) { | |
| 182 DCHECK_GT(min_capture_period_.InMicroseconds(), 0); | |
| 183 } | |
| 184 | |
| 185 void SmoothEventSampler::ConsiderPresentationEvent(base::TimeTicks event_time) { | |
| 186 DCHECK(!event_time.is_null()); | |
| 187 | |
| 188 // Add tokens to the bucket based on advancement in time. Then, re-bound the | |
| 189 // number of tokens in the bucket. Overflow occurs when there is too much | |
| 190 // time between events (a common case), or when RecordSample() is not being | |
| 191 // called often enough (a bug). On the other hand, if RecordSample() is being | |
| 192 // called too often (e.g., as a reaction to IsOverdueForSamplingAt()), the | |
| 193 // bucket will underflow. | |
| 194 if (!current_event_.is_null()) { | |
| 195 if (current_event_ < event_time) { | |
| 196 token_bucket_ += event_time - current_event_; | |
| 197 if (token_bucket_ > token_bucket_capacity_) | |
| 198 token_bucket_ = token_bucket_capacity_; | |
| 199 } | |
| 200 TRACE_COUNTER1("gpu.capture", | |
| 201 "MirroringTokenBucketUsec", | |
| 202 std::max<int64>(0, token_bucket_.InMicroseconds())); | |
| 203 } | |
| 204 current_event_ = event_time; | |
| 205 } | |
| 206 | |
| 207 bool SmoothEventSampler::ShouldSample() const { | |
| 208 return token_bucket_ >= min_capture_period_; | |
| 209 } | |
| 210 | |
| 211 void SmoothEventSampler::RecordSample() { | |
| 212 token_bucket_ -= min_capture_period_; | |
| 213 if (token_bucket_ < base::TimeDelta()) | |
| 214 token_bucket_ = base::TimeDelta(); | |
| 215 TRACE_COUNTER1("gpu.capture", | |
| 216 "MirroringTokenBucketUsec", | |
| 217 std::max<int64>(0, token_bucket_.InMicroseconds())); | |
| 218 | |
| 219 if (HasUnrecordedEvent()) { | |
| 220 last_sample_ = current_event_; | |
| 221 overdue_sample_count_ = 0; | |
| 222 } else { | |
| 223 ++overdue_sample_count_; | |
| 224 } | |
| 225 } | |
| 226 | |
| 227 bool SmoothEventSampler::IsOverdueForSamplingAt(base::TimeTicks event_time) | |
| 228 const { | |
| 229 DCHECK(!event_time.is_null()); | |
| 230 | |
| 231 if (!HasUnrecordedEvent() && overdue_sample_count_ >= redundant_capture_goal_) | |
| 232 return false; // Not dirty. | |
| 233 | |
| 234 if (last_sample_.is_null()) | |
| 235 return true; | |
| 236 | |
| 237 // If we're dirty but not yet old, then we've recently gotten updates, so we | |
| 238 // won't request a sample just yet. | |
| 239 base::TimeDelta dirty_interval = event_time - last_sample_; | |
| 240 return dirty_interval >= | |
| 241 base::TimeDelta::FromMilliseconds(kNonAnimatingThresholdMillis); | |
| 242 } | |
| 243 | |
| 244 bool SmoothEventSampler::HasUnrecordedEvent() const { | |
| 245 return !current_event_.is_null() && current_event_ != last_sample_; | |
| 246 } | |
| 247 | |
| 248 AnimatedContentSampler::AnimatedContentSampler( | 42 AnimatedContentSampler::AnimatedContentSampler( |
| 249 base::TimeDelta min_capture_period) | 43 base::TimeDelta min_capture_period) |
| 250 : min_capture_period_(min_capture_period) {} | 44 : min_capture_period_(min_capture_period) {} |
| 251 | 45 |
| 252 AnimatedContentSampler::~AnimatedContentSampler() {} | 46 AnimatedContentSampler::~AnimatedContentSampler() {} |
| 253 | 47 |
| 254 void AnimatedContentSampler::ConsiderPresentationEvent( | 48 void AnimatedContentSampler::ConsiderPresentationEvent( |
| 255 const gfx::Rect& damage_rect, base::TimeTicks event_time) { | 49 const gfx::Rect& damage_rect, base::TimeTicks event_time) { |
| 256 AddObservation(damage_rect, event_time); | 50 AddObservation(damage_rect, event_time); |
| 257 | 51 |
| (...skipping 161 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 419 if (borrowed_time_ >= min_capture_period_) { | 213 if (borrowed_time_ >= min_capture_period_) { |
| 420 borrowed_time_ -= min_capture_period_; | 214 borrowed_time_ -= min_capture_period_; |
| 421 frame_timestamp_ = base::TimeTicks(); | 215 frame_timestamp_ = base::TimeTicks(); |
| 422 } else { | 216 } else { |
| 423 sequence_offset_ += advancement; | 217 sequence_offset_ += advancement; |
| 424 frame_timestamp_ = timebase + sequence_offset_; | 218 frame_timestamp_ = timebase + sequence_offset_; |
| 425 } | 219 } |
| 426 } | 220 } |
| 427 | 221 |
| 428 } // namespace content | 222 } // namespace content |
| OLD | NEW |