OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import logging | 5 import logging |
6 from telemetry.perf_tests_helper import FlattenList | 6 from telemetry.perf_tests_helper import FlattenList |
7 from telemetry.util import statistics | 7 from telemetry.util import statistics |
| 8 from telemetry.value import improvement_direction |
8 from telemetry.value import list_of_scalar_values | 9 from telemetry.value import list_of_scalar_values |
9 from telemetry.value import scalar | 10 from telemetry.value import scalar |
10 from telemetry.web_perf.metrics import rendering_stats | 11 from telemetry.web_perf.metrics import rendering_stats |
11 from telemetry.web_perf.metrics import timeline_based_metric | 12 from telemetry.web_perf.metrics import timeline_based_metric |
12 | 13 |
13 | 14 |
14 NOT_ENOUGH_FRAMES_MESSAGE = ( | 15 NOT_ENOUGH_FRAMES_MESSAGE = ( |
15 'Not enough frames for smoothness metrics (at least two are required).\n' | 16 'Not enough frames for smoothness metrics (at least two are required).\n' |
16 'Issues that have caused this in the past:\n' | 17 'Issues that have caused this in the past:\n' |
17 '- Browser bugs that prevents the page from redrawing\n' | 18 '- Browser bugs that prevents the page from redrawing\n' |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
117 max_frame_delay = round(max(normalized_frame_lengths)) | 118 max_frame_delay = round(max(normalized_frame_lengths)) |
118 frame_lengths = normalized_frame_lengths | 119 frame_lengths = normalized_frame_lengths |
119 else: | 120 else: |
120 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 121 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
121 | 122 |
122 return ( | 123 return ( |
123 scalar.ScalarValue( | 124 scalar.ScalarValue( |
124 page, 'avg_surface_fps', 'fps', avg_surface_fps, | 125 page, 'avg_surface_fps', 'fps', avg_surface_fps, |
125 description='Average frames per second as measured by the ' | 126 description='Average frames per second as measured by the ' |
126 'platform\'s SurfaceFlinger.', | 127 'platform\'s SurfaceFlinger.', |
127 none_value_reason=none_value_reason), | 128 none_value_reason=none_value_reason, |
| 129 improvement_direction=improvement_direction.UP), |
128 scalar.ScalarValue( | 130 scalar.ScalarValue( |
129 page, 'jank_count', 'janks', jank_count, | 131 page, 'jank_count', 'janks', jank_count, |
130 description='Number of changes in frame rate as measured by the ' | 132 description='Number of changes in frame rate as measured by the ' |
131 'platform\'s SurfaceFlinger.', | 133 'platform\'s SurfaceFlinger.', |
132 none_value_reason=none_value_reason), | 134 none_value_reason=none_value_reason, |
| 135 improvement_direction=improvement_direction.DOWN), |
133 scalar.ScalarValue( | 136 scalar.ScalarValue( |
134 page, 'max_frame_delay', 'vsyncs', max_frame_delay, | 137 page, 'max_frame_delay', 'vsyncs', max_frame_delay, |
135 description='Largest frame time as measured by the platform\'s ' | 138 description='Largest frame time as measured by the platform\'s ' |
136 'SurfaceFlinger.', | 139 'SurfaceFlinger.', |
137 none_value_reason=none_value_reason), | 140 none_value_reason=none_value_reason, |
| 141 improvement_direction=improvement_direction.DOWN), |
138 list_of_scalar_values.ListOfScalarValues( | 142 list_of_scalar_values.ListOfScalarValues( |
139 page, 'frame_lengths', 'vsyncs', frame_lengths, | 143 page, 'frame_lengths', 'vsyncs', frame_lengths, |
140 description='Frame time in vsyncs as measured by the platform\'s ' | 144 description='Frame time in vsyncs as measured by the platform\'s ' |
141 'SurfaceFlinger.', | 145 'SurfaceFlinger.', |
142 none_value_reason=none_value_reason) | 146 none_value_reason=none_value_reason, |
| 147 improvement_direction=improvement_direction.DOWN) |
143 ) | 148 ) |
144 | 149 |
145 def _ComputeLatencyMetric(self, page, stats, name, list_of_latency_lists): | 150 def _ComputeLatencyMetric(self, page, stats, name, list_of_latency_lists): |
146 """Returns Values for the mean and discrepancy for given latency stats.""" | 151 """Returns Values for the mean and discrepancy for given latency stats.""" |
147 mean_latency = None | 152 mean_latency = None |
148 latency_discrepancy = None | 153 latency_discrepancy = None |
149 none_value_reason = None | 154 none_value_reason = None |
150 if self._HasEnoughFrames(stats.frame_timestamps): | 155 if self._HasEnoughFrames(stats.frame_timestamps): |
151 latency_list = FlattenList(list_of_latency_lists) | 156 latency_list = FlattenList(list_of_latency_lists) |
152 if len(latency_list) == 0: | 157 if len(latency_list) == 0: |
153 return () | 158 return () |
154 mean_latency = round(statistics.ArithmeticMean(latency_list), 3) | 159 mean_latency = round(statistics.ArithmeticMean(latency_list), 3) |
155 latency_discrepancy = ( | 160 latency_discrepancy = ( |
156 round(statistics.DurationsDiscrepancy(latency_list), 4)) | 161 round(statistics.DurationsDiscrepancy(latency_list), 4)) |
157 else: | 162 else: |
158 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 163 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
159 return ( | 164 return ( |
160 scalar.ScalarValue( | 165 scalar.ScalarValue( |
161 page, 'mean_%s' % name, 'ms', mean_latency, | 166 page, 'mean_%s' % name, 'ms', mean_latency, |
162 description='Arithmetic mean of the raw %s values' % name, | 167 description='Arithmetic mean of the raw %s values' % name, |
163 none_value_reason=none_value_reason), | 168 none_value_reason=none_value_reason, |
| 169 improvement_direction=improvement_direction.DOWN), |
164 scalar.ScalarValue( | 170 scalar.ScalarValue( |
165 page, '%s_discrepancy' % name, 'ms', latency_discrepancy, | 171 page, '%s_discrepancy' % name, 'ms', latency_discrepancy, |
166 description='Discrepancy of the raw %s values' % name, | 172 description='Discrepancy of the raw %s values' % name, |
167 none_value_reason=none_value_reason) | 173 none_value_reason=none_value_reason, |
| 174 improvement_direction=improvement_direction.DOWN) |
168 ) | 175 ) |
169 | 176 |
170 def _ComputeFirstGestureScrollUpdateLatency(self, page, stats): | 177 def _ComputeFirstGestureScrollUpdateLatency(self, page, stats): |
171 """Returns a Value for the first gesture scroll update latency.""" | 178 """Returns a Value for the first gesture scroll update latency.""" |
172 first_gesture_scroll_update_latency = None | 179 first_gesture_scroll_update_latency = None |
173 none_value_reason = None | 180 none_value_reason = None |
174 if self._HasEnoughFrames(stats.frame_timestamps): | 181 if self._HasEnoughFrames(stats.frame_timestamps): |
175 latency_list = FlattenList(stats.gesture_scroll_update_latency) | 182 latency_list = FlattenList(stats.gesture_scroll_update_latency) |
176 if len(latency_list) == 0: | 183 if len(latency_list) == 0: |
177 return () | 184 return () |
178 first_gesture_scroll_update_latency = round(latency_list[0], 4) | 185 first_gesture_scroll_update_latency = round(latency_list[0], 4) |
179 else: | 186 else: |
180 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 187 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
181 return ( | 188 return ( |
182 scalar.ScalarValue( | 189 scalar.ScalarValue( |
183 page, 'first_gesture_scroll_update_latency', 'ms', | 190 page, 'first_gesture_scroll_update_latency', 'ms', |
184 first_gesture_scroll_update_latency, | 191 first_gesture_scroll_update_latency, |
185 description='First gesture scroll update latency measures the time it ' | 192 description='First gesture scroll update latency measures the time it ' |
186 'takes to process the very first gesture scroll update ' | 193 'takes to process the very first gesture scroll update ' |
187 'input event. The first scroll gesture can often get ' | 194 'input event. The first scroll gesture can often get ' |
188 'delayed by work related to page loading.', | 195 'delayed by work related to page loading.', |
189 none_value_reason=none_value_reason), | 196 none_value_reason=none_value_reason, |
| 197 improvement_direction=improvement_direction.DOWN), |
190 ) | 198 ) |
191 | 199 |
192 def _ComputeQueueingDuration(self, page, stats): | 200 def _ComputeQueueingDuration(self, page, stats): |
193 """Returns a Value for the frame queueing durations.""" | 201 """Returns a Value for the frame queueing durations.""" |
194 queueing_durations = None | 202 queueing_durations = None |
195 none_value_reason = None | 203 none_value_reason = None |
196 if 'frame_queueing_durations' in stats.errors: | 204 if 'frame_queueing_durations' in stats.errors: |
197 none_value_reason = stats.errors['frame_queueing_durations'] | 205 none_value_reason = stats.errors['frame_queueing_durations'] |
198 elif self._HasEnoughFrames(stats.frame_timestamps): | 206 elif self._HasEnoughFrames(stats.frame_timestamps): |
199 queueing_durations = FlattenList(stats.frame_queueing_durations) | 207 queueing_durations = FlattenList(stats.frame_queueing_durations) |
200 if len(queueing_durations) == 0: | 208 if len(queueing_durations) == 0: |
201 queueing_durations = None | 209 queueing_durations = None |
202 none_value_reason = 'No frame queueing durations recorded.' | 210 none_value_reason = 'No frame queueing durations recorded.' |
203 else: | 211 else: |
204 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 212 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
205 return list_of_scalar_values.ListOfScalarValues( | 213 return list_of_scalar_values.ListOfScalarValues( |
206 page, 'queueing_durations', 'ms', queueing_durations, | 214 page, 'queueing_durations', 'ms', queueing_durations, |
207 description='The frame queueing duration quantifies how out of sync ' | 215 description='The frame queueing duration quantifies how out of sync ' |
208 'the compositor and renderer threads are. It is the amount ' | 216 'the compositor and renderer threads are. It is the amount ' |
209 'of wall time that elapses between a ' | 217 'of wall time that elapses between a ' |
210 'ScheduledActionSendBeginMainFrame event in the compositor ' | 218 'ScheduledActionSendBeginMainFrame event in the compositor ' |
211 'thread and the corresponding BeginMainFrame event in the ' | 219 'thread and the corresponding BeginMainFrame event in the ' |
212 'main thread.', | 220 'main thread.', |
213 none_value_reason=none_value_reason) | 221 none_value_reason=none_value_reason, |
| 222 improvement_direction=improvement_direction.DOWN) |
214 | 223 |
215 def _ComputeFrameTimeMetric(self, page, stats): | 224 def _ComputeFrameTimeMetric(self, page, stats): |
216 """Returns Values for the frame time metrics. | 225 """Returns Values for the frame time metrics. |
217 | 226 |
218 This includes the raw and mean frame times, as well as the percentage of | 227 This includes the raw and mean frame times, as well as the percentage of |
219 frames that were hitting 60 fps. | 228 frames that were hitting 60 fps. |
220 """ | 229 """ |
221 frame_times = None | 230 frame_times = None |
222 mean_frame_time = None | 231 mean_frame_time = None |
223 percentage_smooth = None | 232 percentage_smooth = None |
224 none_value_reason = None | 233 none_value_reason = None |
225 if self._HasEnoughFrames(stats.frame_timestamps): | 234 if self._HasEnoughFrames(stats.frame_timestamps): |
226 frame_times = FlattenList(stats.frame_times) | 235 frame_times = FlattenList(stats.frame_times) |
227 mean_frame_time = round(statistics.ArithmeticMean(frame_times), 3) | 236 mean_frame_time = round(statistics.ArithmeticMean(frame_times), 3) |
228 # We use 17ms as a somewhat looser threshold, instead of 1000.0/60.0. | 237 # We use 17ms as a somewhat looser threshold, instead of 1000.0/60.0. |
229 smooth_threshold = 17.0 | 238 smooth_threshold = 17.0 |
230 smooth_count = sum(1 for t in frame_times if t < smooth_threshold) | 239 smooth_count = sum(1 for t in frame_times if t < smooth_threshold) |
231 percentage_smooth = float(smooth_count) / len(frame_times) * 100.0 | 240 percentage_smooth = float(smooth_count) / len(frame_times) * 100.0 |
232 else: | 241 else: |
233 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 242 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
234 return ( | 243 return ( |
235 list_of_scalar_values.ListOfScalarValues( | 244 list_of_scalar_values.ListOfScalarValues( |
236 page, 'frame_times', 'ms', frame_times, | 245 page, 'frame_times', 'ms', frame_times, |
237 description='List of raw frame times, helpful to understand the ' | 246 description='List of raw frame times, helpful to understand the ' |
238 'other metrics.', | 247 'other metrics.', |
239 none_value_reason=none_value_reason), | 248 none_value_reason=none_value_reason, |
| 249 improvement_direction=improvement_direction.DOWN), |
240 scalar.ScalarValue( | 250 scalar.ScalarValue( |
241 page, 'mean_frame_time', 'ms', mean_frame_time, | 251 page, 'mean_frame_time', 'ms', mean_frame_time, |
242 description='Arithmetic mean of frame times.', | 252 description='Arithmetic mean of frame times.', |
243 none_value_reason=none_value_reason), | 253 none_value_reason=none_value_reason, |
| 254 improvement_direction=improvement_direction.DOWN), |
244 scalar.ScalarValue( | 255 scalar.ScalarValue( |
245 page, 'percentage_smooth', 'score', percentage_smooth, | 256 page, 'percentage_smooth', 'score', percentage_smooth, |
246 description='Percentage of frames that were hitting 60 fps.', | 257 description='Percentage of frames that were hitting 60 fps.', |
247 none_value_reason=none_value_reason) | 258 none_value_reason=none_value_reason, |
| 259 improvement_direction=improvement_direction.DOWN) |
248 ) | 260 ) |
249 | 261 |
250 def _ComputeFrameTimeDiscrepancy(self, page, stats): | 262 def _ComputeFrameTimeDiscrepancy(self, page, stats): |
251 """Returns a Value for the absolute discrepancy of frame time stamps.""" | 263 """Returns a Value for the absolute discrepancy of frame time stamps.""" |
252 | 264 |
253 frame_discrepancy = None | 265 frame_discrepancy = None |
254 none_value_reason = None | 266 none_value_reason = None |
255 if self._HasEnoughFrames(stats.frame_timestamps): | 267 if self._HasEnoughFrames(stats.frame_timestamps): |
256 frame_discrepancy = round(statistics.TimestampsDiscrepancy( | 268 frame_discrepancy = round(statistics.TimestampsDiscrepancy( |
257 stats.frame_timestamps), 4) | 269 stats.frame_timestamps), 4) |
258 else: | 270 else: |
259 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 271 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
260 return scalar.ScalarValue( | 272 return scalar.ScalarValue( |
261 page, 'frame_time_discrepancy', 'ms', frame_discrepancy, | 273 page, 'frame_time_discrepancy', 'ms', frame_discrepancy, |
262 description='Absolute discrepancy of frame time stamps, where ' | 274 description='Absolute discrepancy of frame time stamps, where ' |
263 'discrepancy is a measure of irregularity. It quantifies ' | 275 'discrepancy is a measure of irregularity. It quantifies ' |
264 'the worst jank. For a single pause, discrepancy ' | 276 'the worst jank. For a single pause, discrepancy ' |
265 'corresponds to the length of this pause in milliseconds. ' | 277 'corresponds to the length of this pause in milliseconds. ' |
266 'Consecutive pauses increase the discrepancy. This metric ' | 278 'Consecutive pauses increase the discrepancy. This metric ' |
267 'is important because even if the mean and 95th ' | 279 'is important because even if the mean and 95th ' |
268 'percentile are good, one long pause in the middle of an ' | 280 'percentile are good, one long pause in the middle of an ' |
269 'interaction is still bad.', | 281 'interaction is still bad.', |
270 none_value_reason=none_value_reason) | 282 none_value_reason=none_value_reason, |
| 283 improvement_direction=improvement_direction.DOWN) |
271 | 284 |
272 def _ComputeMeanPixelsApproximated(self, page, stats): | 285 def _ComputeMeanPixelsApproximated(self, page, stats): |
273 """Add the mean percentage of pixels approximated. | 286 """Add the mean percentage of pixels approximated. |
274 | 287 |
275 This looks at tiles which are missing or of low or non-ideal resolution. | 288 This looks at tiles which are missing or of low or non-ideal resolution. |
276 """ | 289 """ |
277 mean_pixels_approximated = None | 290 mean_pixels_approximated = None |
278 none_value_reason = None | 291 none_value_reason = None |
279 if self._HasEnoughFrames(stats.frame_timestamps): | 292 if self._HasEnoughFrames(stats.frame_timestamps): |
280 mean_pixels_approximated = round(statistics.ArithmeticMean( | 293 mean_pixels_approximated = round(statistics.ArithmeticMean( |
281 FlattenList(stats.approximated_pixel_percentages)), 3) | 294 FlattenList(stats.approximated_pixel_percentages)), 3) |
282 else: | 295 else: |
283 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE | 296 none_value_reason = NOT_ENOUGH_FRAMES_MESSAGE |
284 return scalar.ScalarValue( | 297 return scalar.ScalarValue( |
285 page, 'mean_pixels_approximated', 'percent', mean_pixels_approximated, | 298 page, 'mean_pixels_approximated', 'percent', mean_pixels_approximated, |
286 description='Percentage of pixels that were approximated ' | 299 description='Percentage of pixels that were approximated ' |
287 '(checkerboarding, low-resolution tiles, etc.).', | 300 '(checkerboarding, low-resolution tiles, etc.).', |
288 none_value_reason=none_value_reason) | 301 none_value_reason=none_value_reason, |
| 302 improvement_direction=improvement_direction.DOWN) |
OLD | NEW |