OLD | NEW |
(Empty) | |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. |
| 4 |
| 5 import logging |
| 6 |
| 7 from telemetry.util import statistics |
| 8 |
| 9 DISPLAY_HERTZ = 60.0 |
| 10 VSYNC_DURATION = 1e6 / DISPLAY_HERTZ |
| 11 # When to consider a frame frozen (in VSYNC units): meaning 1 initial |
| 12 # frame + 5 repeats of that frame. |
| 13 FROZEN_THRESHOLD = 6 |
| 14 # Severity factor. |
| 15 SEVERITY = 3 |
| 16 |
| 17 IDEAL_RENDER_INSTANT = 'Ideal Render Instant' |
| 18 ACTUAL_RENDER_BEGIN = 'Actual Render Begin' |
| 19 ACTUAL_RENDER_END = 'Actual Render End' |
| 20 SERIAL = 'Serial' |
| 21 |
| 22 |
| 23 class TimeStats(object): |
| 24 """Stats container for webrtc rendering metrics.""" |
| 25 |
| 26 def __init__(self, drift_time=None, mean_drift_time=None, |
| 27 std_dev_drift_time=None, percent_badly_out_of_sync=None, |
| 28 percent_out_of_sync=None, smoothness_score=None, freezing_score=None, |
| 29 rendering_length_error=None, fps=None, frame_distribution=None): |
| 30 self.drift_time = drift_time |
| 31 self.mean_drift_time = mean_drift_time |
| 32 self.std_dev_drift_time = std_dev_drift_time |
| 33 self.percent_badly_out_of_sync = percent_badly_out_of_sync |
| 34 self.percent_out_of_sync = percent_out_of_sync |
| 35 self.smoothness_score = smoothness_score |
| 36 self.freezing_score = freezing_score |
| 37 self.rendering_length_error = rendering_length_error |
| 38 self.fps = fps |
| 39 self.frame_distribution = frame_distribution |
| 40 |
| 41 |
| 42 |
| 43 class WebMediaPlayerMsRenderingStats(object): |
| 44 """Analyzes events of WebMediaPlayerMs type.""" |
| 45 |
| 46 def __init__(self, events): |
| 47 """Save relevant events according to their stream.""" |
| 48 self.stream_to_events = self._MapEventsToStream(events) |
| 49 |
| 50 def _IsEventValid(self, event): |
| 51 """Check that the needed arguments are present in event. |
| 52 |
| 53 Args: |
| 54 event: event to check. |
| 55 |
| 56 Returns: |
| 57 True is event is valid, false otherwise.""" |
| 58 if not event.args: |
| 59 return False |
| 60 mandatory = [ACTUAL_RENDER_BEGIN, ACTUAL_RENDER_END, |
| 61 IDEAL_RENDER_INSTANT, SERIAL] |
| 62 for parameter in mandatory: |
| 63 if not parameter in event.args: |
| 64 return False |
| 65 return True |
| 66 |
| 67 def _MapEventsToStream(self, events): |
| 68 """Build a dictionary of events indexed by stream. |
| 69 |
| 70 The events of interest have a 'Serial' argument which represents the |
| 71 stream ID. The 'Serial' argument identifies the local or remote nature of |
| 72 the stream with a least significant bit of 0 or 1 as well as the hash |
| 73 value of the video track's URL. So stream::=hash(0|1} . The method will |
| 74 then list the events of the same stream in a frame_distribution on stream |
| 75 id. Practically speaking remote streams have an odd stream id and local |
| 76 streams have a even stream id. |
| 77 Args: |
| 78 events: Telemetry WebMediaPlayerMs events. |
| 79 |
| 80 Returns: |
| 81 A dict of stream IDs mapped to events on that stream. |
| 82 """ |
| 83 stream_to_events = {} |
| 84 for event in events: |
| 85 if not self._IsEventValid(event): |
| 86 # This is not a render event, skip it. |
| 87 continue |
| 88 stream = event.args[SERIAL] |
| 89 events_for_stream = stream_to_events.setdefault(stream, []) |
| 90 events_for_stream.append(event) |
| 91 |
| 92 return stream_to_events |
| 93 |
| 94 def _GetCadence(self, relevant_events): |
| 95 """Calculate the apparent cadence of the rendering. |
| 96 |
| 97 In this paragraph I will be using regex notation. What is intended by the |
| 98 word cadence is a sort of extended instantaneous 'Cadence' (thus not |
| 99 necessarily periodic). Just as an example, a normal 'Cadence' could be |
| 100 something like [2 3] which means possibly an observed frame persistance |
| 101 progression of [{2 3}+] for an ideal 20FPS video source. So what we are |
| 102 calculating here is the list of frame persistance, kind of a |
| 103 'Proto-Cadence', but cadence is shorter so we abuse the word. |
| 104 |
| 105 Args: |
| 106 relevant_events: list of Telemetry events. |
| 107 |
| 108 Returns: |
| 109 a list of frame persistance values. |
| 110 """ |
| 111 cadence = [] |
| 112 frame_persistence = 0 |
| 113 old_ideal_render = 0 |
| 114 for event in relevant_events: |
| 115 if not self._IsEventValid(event): |
| 116 # This event is not a render event so skip it. |
| 117 continue |
| 118 if event.args[IDEAL_RENDER_INSTANT] == old_ideal_render: |
| 119 frame_persistence += 1 |
| 120 else: |
| 121 cadence.append(frame_persistence) |
| 122 frame_persistence = 1 |
| 123 old_ideal_render = event.args[IDEAL_RENDER_INSTANT] |
| 124 cadence.append(frame_persistence) |
| 125 cadence.pop(0) |
| 126 return cadence |
| 127 |
| 128 def _GetSourceToOutputDistribution(self, cadence): |
| 129 """Create distribution for the cadence frame display values. |
| 130 |
| 131 If the overall display distribution is A1:A2:..:An, this will tell us how |
| 132 many times a frame stays displayed during Ak*VSYNC_DURATION, also known as |
| 133 'source to output' distribution. Or in other terms: |
| 134 a distribution B::= let C be the cadence, B[k]=p with k in Unique(C) |
| 135 and p=Card(k in C). |
| 136 |
| 137 Args: |
| 138 cadence: list of frame persistance values. |
| 139 |
| 140 Returns: |
| 141 a dictionary containing the distribution |
| 142 """ |
| 143 frame_distribution = {} |
| 144 for ticks in cadence: |
| 145 ticks_so_far = frame_distribution.setdefault(ticks, 0) |
| 146 frame_distribution[ticks] = ticks_so_far + 1 |
| 147 return frame_distribution |
| 148 |
| 149 def _GetFpsFromCadence(self, frame_distribution): |
| 150 """Calculate the apparent FPS from frame distribution. |
| 151 |
| 152 Knowing the display frequency and the frame distribution, it is possible to |
| 153 calculate the video apparent frame rate as played by WebMediaPlayerMs |
| 154 module. |
| 155 |
| 156 Args: |
| 157 frame_distribution: the source to output distribution. |
| 158 |
| 159 Returns: |
| 160 the video apparent frame rate. |
| 161 """ |
| 162 number_frames = sum(frame_distribution.values()) |
| 163 number_vsyncs = sum([ticks * frame_distribution[ticks] |
| 164 for ticks in frame_distribution]) |
| 165 mean_ratio = float(number_vsyncs) / number_frames |
| 166 return DISPLAY_HERTZ / mean_ratio |
| 167 |
| 168 def _GetFrozenFramesReports(self, frame_distribution): |
| 169 """Find evidence of frozen frames in distribution. |
| 170 |
| 171 For simplicity we count as freezing the frames that appear at least five |
| 172 times in a row counted from 'Ideal Render Instant' perspective. So let's |
| 173 say for 1 source frame, we rendered 6 frames, then we consider 5 of these |
| 174 rendered frames as frozen. But we mitigate this by saying anything under |
| 175 5 frozen frames will not be counted as frozen. |
| 176 |
| 177 Args: |
| 178 frame_distribution: the source to output distribution. |
| 179 |
| 180 Returns: |
| 181 a list of dicts whose keys are ('frozen_frames', 'occurrences'). |
| 182 """ |
| 183 frozen_frames = [] |
| 184 frozen_frame_vsyncs = [ticks for ticks in frame_distribution if ticks >= |
| 185 FROZEN_THRESHOLD] |
| 186 for frozen_frames_vsync in frozen_frame_vsyncs: |
| 187 logging.debug('%s frames not updated after %s vsyncs', |
| 188 frame_distribution[frozen_frames_vsync], frozen_frames_vsync) |
| 189 frozen_frames.append( |
| 190 {'frozen_frames': frozen_frames_vsync - 1, |
| 191 'occurrences': frame_distribution[frozen_frames_vsync]}) |
| 192 return frozen_frames |
| 193 |
| 194 def _FrozenPenaltyWeight(self, number_frozen_frames): |
| 195 """Returns the weighted penalty for a number of frozen frames. |
| 196 |
| 197 As mentioned earlier, we count for frozen anything above 6 vsync display |
| 198 duration for the same 'Initial Render Instant', which is five frozen |
| 199 frames. |
| 200 |
| 201 Args: |
| 202 number_frozen_frames: number of frozen frames. |
| 203 |
| 204 Returns: |
| 205 the penalty weight (int) for that number of frozen frames. |
| 206 """ |
| 207 |
| 208 penalty = { |
| 209 0: 0, |
| 210 1: 0, |
| 211 2: 0, |
| 212 3: 0, |
| 213 4: 0, |
| 214 5: 1, |
| 215 6: 5, |
| 216 7: 15, |
| 217 8: 25 |
| 218 } |
| 219 weight = penalty.get(number_frozen_frames, 8 * (number_frozen_frames - 4)) |
| 220 return weight |
| 221 |
| 222 def _IsRemoteStream(self, stream): |
| 223 """Check if stream is remote.""" |
| 224 return stream % 2 |
| 225 |
| 226 def _GetDrifTimeStats(self, relevant_events, cadence): |
| 227 """Get the drift time statistics. |
| 228 |
| 229 This method will calculate drift_time stats, that is to say : |
| 230 drift_time::= list(actual render begin - ideal render). |
| 231 rendering_length error::= the rendering length error. |
| 232 |
| 233 Args: |
| 234 relevant_events: events to get drift times stats from. |
| 235 cadence: list of frame persistence values. |
| 236 |
| 237 Returns: |
| 238 a tuple of (drift_time, rendering_length_error). |
| 239 """ |
| 240 drift_time = [] |
| 241 old_ideal_render = 0 |
| 242 discrepancy = [] |
| 243 index = 0 |
| 244 for event in relevant_events: |
| 245 current_ideal_render = event.args[IDEAL_RENDER_INSTANT] |
| 246 if current_ideal_render == old_ideal_render: |
| 247 # Skip to next event because we're looking for a source frame. |
| 248 continue |
| 249 actual_render_begin = event.args[ACTUAL_RENDER_BEGIN] |
| 250 drift_time.append(actual_render_begin - current_ideal_render) |
| 251 discrepancy.append(abs(current_ideal_render - old_ideal_render |
| 252 - VSYNC_DURATION * cadence[index])) |
| 253 old_ideal_render = current_ideal_render |
| 254 index += 1 |
| 255 discrepancy.pop(0) |
| 256 last_ideal_render = relevant_events[-1].args[IDEAL_RENDER_INSTANT] |
| 257 first_ideal_render = relevant_events[0].args[IDEAL_RENDER_INSTANT] |
| 258 rendering_length_error = 100.0 * (sum([x for x in discrepancy]) / |
| 259 (last_ideal_render - first_ideal_render)) |
| 260 |
| 261 return drift_time, rendering_length_error |
| 262 |
| 263 def _GetSmoothnessStats(self, norm_drift_time): |
| 264 """Get the smoothness stats from the normalized drift time. |
| 265 |
| 266 This method will calculate the smoothness score, along with the percentage |
| 267 of frames badly out of sync and the percentage of frames out of sync. To be |
| 268 considered badly out of sync, a frame has to have missed rendering by at |
| 269 least 2*VSYNC_DURATION. To be considered out of sync, a frame has to have |
| 270 missed rendering by at least one VSYNC_DURATION. |
| 271 The smoothness score is a measure of how out of sync the frames are. |
| 272 |
| 273 Args: |
| 274 norm_drift_time: normalized drift time. |
| 275 |
| 276 Returns: |
| 277 a tuple of (percent_badly_oos, percent_out_of_sync, smoothness_score) |
| 278 """ |
| 279 # How many times is a frame later/earlier than T=2*VSYNC_DURATION. Time is |
| 280 # in microseconds. |
| 281 frames_severely_out_of_sync = len( |
| 282 [x for x in norm_drift_time if abs(x) > 2 * VSYNC_DURATION]) |
| 283 percent_badly_oos = ( |
| 284 100.0 * frames_severely_out_of_sync / len(norm_drift_time)) |
| 285 |
| 286 # How many times is a frame later/earlier than VSYNC_DURATION. |
| 287 frames_out_of_sync = len( |
| 288 [x for x in norm_drift_time if abs(x) > VSYNC_DURATION]) |
| 289 percent_out_of_sync = ( |
| 290 100.0 * frames_out_of_sync / len(norm_drift_time)) |
| 291 |
| 292 frames_oos_only_once = frames_out_of_sync - frames_severely_out_of_sync |
| 293 |
| 294 # Calculate smoothness metric. From the formula, we can see that smoothness |
| 295 # score can be negative. |
| 296 smoothness_score = 100.0 - 100.0 * (frames_oos_only_once + |
| 297 SEVERITY * frames_severely_out_of_sync) / len(norm_drift_time) |
| 298 |
| 299 # Minimum smoothness_score value allowed is zero. |
| 300 if smoothness_score < 0: |
| 301 smoothness_score = 0 |
| 302 |
| 303 return (percent_badly_oos, percent_out_of_sync, smoothness_score) |
| 304 |
| 305 def _GetFreezingScore(self, frame_distribution): |
| 306 """Get the freezing score.""" |
| 307 |
| 308 # The freezing score is based on the source to output distribution. |
| 309 number_vsyncs = sum([n * frame_distribution[n] |
| 310 for n in frame_distribution]) |
| 311 frozen_frames = self._GetFrozenFramesReports(frame_distribution) |
| 312 |
| 313 # Calculate freezing metric. |
| 314 # Freezing metric can be negative if things are really bad. In that case we |
| 315 # change it to zero as minimum valud. |
| 316 freezing_score = 100.0 |
| 317 for frozen_report in frozen_frames: |
| 318 weight = self._FrozenPenaltyWeight(frozen_report['frozen_frames']) |
| 319 freezing_score -= ( |
| 320 100.0 * frozen_report['occurrences'] / number_vsyncs * weight) |
| 321 if freezing_score < 0: |
| 322 freezing_score = 0 |
| 323 |
| 324 return freezing_score |
| 325 |
| 326 def GetTimeStats(self): |
| 327 """Calculate time stamp stats for all remote stream events.""" |
| 328 stats = {} |
| 329 for stream, relevant_events in self.stream_to_events.iteritems(): |
| 330 if len(relevant_events) == 1: |
| 331 logging.debug('Found a stream=%s with just one event', stream) |
| 332 continue |
| 333 if not self._IsRemoteStream(stream): |
| 334 logging.info('Skipping processing of local stream: %s', stream) |
| 335 continue |
| 336 |
| 337 cadence = self._GetCadence(relevant_events) |
| 338 frame_distribution = self._GetSourceToOutputDistribution(cadence) |
| 339 fps = self._GetFpsFromCadence(frame_distribution) |
| 340 |
| 341 drift_time_stats = self._GetDrifTimeStats(relevant_events, cadence) |
| 342 (drift_time, rendering_length_error) = drift_time_stats |
| 343 |
| 344 # Drift time normalization. |
| 345 mean_drift_time = statistics.ArithmeticMean(drift_time) |
| 346 norm_drift_time = [abs(x - mean_drift_time) for x in drift_time] |
| 347 |
| 348 smoothness_stats = self._GetSmoothnessStats(norm_drift_time) |
| 349 (percent_badly_oos, percent_out_of_sync, |
| 350 smoothness_score) = smoothness_stats |
| 351 |
| 352 freezing_score = self._GetFreezingScore(frame_distribution) |
| 353 |
| 354 stats = TimeStats(drift_time=drift_time, |
| 355 percent_badly_out_of_sync=percent_badly_oos, |
| 356 percent_out_of_sync=percent_out_of_sync, |
| 357 smoothness_score=smoothness_score, freezing_score=freezing_score, |
| 358 rendering_length_error=rendering_length_error, fps=fps, |
| 359 frame_distribution=frame_distribution) |
| 360 return stats |
OLD | NEW |