| OLD | NEW |
| 1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
| 2 # Copyright (C) 2012 Zoltan Horvath, Adobe Systems Incorporated. All rights rese
rved. | 2 # Copyright (C) 2012 Zoltan Horvath, Adobe Systems Incorporated. All rights rese
rved. |
| 3 # | 3 # |
| 4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
| 5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
| 6 # met: | 6 # met: |
| 7 # | 7 # |
| 8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
| 9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
| 10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 42 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWrite
r | 42 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWrite
r |
| 43 from webkitpy.layout_tests.port.driver import DriverInput | 43 from webkitpy.layout_tests.port.driver import DriverInput |
| 44 from webkitpy.layout_tests.port.driver import DriverOutput | 44 from webkitpy.layout_tests.port.driver import DriverOutput |
| 45 | 45 |
| 46 DEFAULT_TEST_RUNNER_COUNT = 4 | 46 DEFAULT_TEST_RUNNER_COUNT = 4 |
| 47 | 47 |
| 48 _log = logging.getLogger(__name__) | 48 _log = logging.getLogger(__name__) |
| 49 | 49 |
| 50 | 50 |
| 51 class PerfTestMetric(object): | 51 class PerfTestMetric(object): |
| 52 |
| 52 def __init__(self, metric, unit=None, iterations=None): | 53 def __init__(self, metric, unit=None, iterations=None): |
| 53 # FIXME: Fix runner.js to report correct metric names | 54 # FIXME: Fix runner.js to report correct metric names |
| 54 self._iterations = iterations or [] | 55 self._iterations = iterations or [] |
| 55 self._unit = unit or self.metric_to_unit(metric) | 56 self._unit = unit or self.metric_to_unit(metric) |
| 56 self._metric = self.time_unit_to_metric(self._unit) if metric == 'Time'
else metric | 57 self._metric = self.time_unit_to_metric(self._unit) if metric == 'Time'
else metric |
| 57 | 58 |
| 58 def name(self): | 59 def name(self): |
| 59 return self._metric | 60 return self._metric |
| 60 | 61 |
| 61 def has_values(self): | 62 def has_values(self): |
| (...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 126 if should_log and self._description: | 127 if should_log and self._description: |
| 127 _log.info('DESCRIPTION: %s' % self._description) | 128 _log.info('DESCRIPTION: %s' % self._description) |
| 128 | 129 |
| 129 results = {} | 130 results = {} |
| 130 for metric_name in self._ordered_metrics_name: | 131 for metric_name in self._ordered_metrics_name: |
| 131 metric = self._metrics[metric_name] | 132 metric = self._metrics[metric_name] |
| 132 results[metric.name()] = metric.grouped_iteration_values() | 133 results[metric.name()] = metric.grouped_iteration_values() |
| 133 if should_log: | 134 if should_log: |
| 134 legacy_chromium_bot_compatible_name = self.test_name_without_fil
e_extension().replace('/', ': ') | 135 legacy_chromium_bot_compatible_name = self.test_name_without_fil
e_extension().replace('/', ': ') |
| 135 self.log_statistics(legacy_chromium_bot_compatible_name + ': ' +
metric.name(), | 136 self.log_statistics(legacy_chromium_bot_compatible_name + ': ' +
metric.name(), |
| 136 metric.flattened_iteration_values(), metric.unit()) | 137 metric.flattened_iteration_values(), metric.
unit()) |
| 137 | 138 |
| 138 return results | 139 return results |
| 139 | 140 |
| 140 @staticmethod | 141 @staticmethod |
| 141 def log_statistics(test_name, values, unit): | 142 def log_statistics(test_name, values, unit): |
| 142 sorted_values = sorted(values) | 143 sorted_values = sorted(values) |
| 143 | 144 |
| 144 # Compute the mean and variance using Knuth's online algorithm (has good
numerical stability). | 145 # Compute the mean and variance using Knuth's online algorithm (has good
numerical stability). |
| 145 square_sum = 0 | 146 square_sum = 0 |
| 146 mean = 0 | 147 mean = 0 |
| 147 for i, time in enumerate(sorted_values): | 148 for i, time in enumerate(sorted_values): |
| 148 delta = time - mean | 149 delta = time - mean |
| 149 sweep = i + 1.0 | 150 sweep = i + 1.0 |
| 150 mean += delta / sweep | 151 mean += delta / sweep |
| 151 square_sum += delta * (time - mean) | 152 square_sum += delta * (time - mean) |
| 152 | 153 |
| 153 middle = int(len(sorted_values) / 2) | 154 middle = int(len(sorted_values) / 2) |
| 154 mean = sum(sorted_values) / len(values) | 155 mean = sum(sorted_values) / len(values) |
| 155 median = sorted_values[middle] if len(sorted_values) % 2 else (sorted_va
lues[middle - 1] + sorted_values[middle]) / 2 | 156 median = sorted_values[middle] if len(sorted_values) % 2 else (sorted_va
lues[middle - 1] + sorted_values[middle]) / 2 |
| 156 stdev = math.sqrt(square_sum / (len(sorted_values) - 1)) if len(sorted_v
alues) > 1 else 0 | 157 stdev = math.sqrt(square_sum / (len(sorted_values) - 1)) if len(sorted_v
alues) > 1 else 0 |
| 157 | 158 |
| 158 _log.info('RESULT %s= %s %s' % (test_name, mean, unit)) | 159 _log.info('RESULT %s= %s %s' % (test_name, mean, unit)) |
| 159 _log.info('median= %s %s, stdev= %s %s, min= %s %s, max= %s %s' % | 160 _log.info('median= %s %s, stdev= %s %s, min= %s %s, max= %s %s' % |
| 160 (median, unit, stdev, unit, sorted_values[0], unit, sorted_values[-1
], unit)) | 161 (median, unit, stdev, unit, sorted_values[0], unit, sorted_val
ues[-1], unit)) |
| 161 | 162 |
| 162 _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IG
NORECASE) | 163 _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IG
NORECASE) |
| 163 _metrics_regex = re.compile(r'^(?P<metric>Time|Malloc|JS Heap):') | 164 _metrics_regex = re.compile(r'^(?P<metric>Time|Malloc|JS Heap):') |
| 164 _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values'
] | 165 _statistics_keys = ['avg', 'median', 'stdev', 'min', 'max', 'unit', 'values'
] |
| 165 _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s
+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)') | 166 _score_regex = re.compile(r'^(?P<key>' + r'|'.join(_statistics_keys) + r')\s
+(?P<value>([0-9\.]+(,\s+)?)+)\s*(?P<unit>.*)') |
| 166 _console_regex = re.compile(r'^CONSOLE (MESSAGE|WARNING):') | 167 _console_regex = re.compile(r'^CONSOLE (MESSAGE|WARNING):') |
| 167 | 168 |
| 168 def _run_with_driver(self, driver, time_out_ms): | 169 def _run_with_driver(self, driver, time_out_ms): |
| 169 output = self.run_single(driver, self.test_path(), time_out_ms) | 170 output = self.run_single(driver, self.test_path(), time_out_ms) |
| 170 self._filter_output(output) | 171 self._filter_output(output) |
| (...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 244 re.compile(r'^\s*Running \d+ times$'), | 245 re.compile(r'^\s*Running \d+ times$'), |
| 245 re.compile(r'^\s*Ignoring warm-up '), | 246 re.compile(r'^\s*Ignoring warm-up '), |
| 246 re.compile(r'^\s*Info:'), | 247 re.compile(r'^\s*Info:'), |
| 247 re.compile(r'^\s*\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'), | 248 re.compile(r'^\s*\d+(.\d+)?(\s*(runs\/s|ms|fps))?$'), |
| 248 # Following are for handle existing test like Dromaeo | 249 # Following are for handle existing test like Dromaeo |
| 249 re.compile(re.escape("""main frame - has 1 onunload handler(s)""")), | 250 re.compile(re.escape("""main frame - has 1 onunload handler(s)""")), |
| 250 re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1
onunload handler(s)""")), | 251 re.compile(re.escape("""frame "<!--framePath //<!--frame0-->-->" - has 1
onunload handler(s)""")), |
| 251 re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-
->-->" - has 1 onunload handler(s)""")), | 252 re.compile(re.escape("""frame "<!--framePath //<!--frame0-->/<!--frame0-
->-->" - has 1 onunload handler(s)""")), |
| 252 # Following is for html5.html | 253 # Following is for html5.html |
| 253 re.compile(re.escape("""Blocked access to external URL http://www.whatwg
.org/specs/web-apps/current-work/""")), | 254 re.compile(re.escape("""Blocked access to external URL http://www.whatwg
.org/specs/web-apps/current-work/""")), |
| 254 re.compile(r"CONSOLE MESSAGE: (line \d+: )?Blocked script execution in '
[A-Za-z0-9\-\.:]+' because the document's frame is sandboxed and the 'allow-scri
pts' permission is not set."), | 255 re.compile( |
| 256 r"CONSOLE MESSAGE: (line \d+: )?Blocked script execution in '[A-Za-z
0-9\-\.:]+' because the document's frame is sandboxed and the 'allow-scripts' pe
rmission is not set."), |
| 255 re.compile(r"CONSOLE MESSAGE: (line \d+: )?Not allowed to load local res
ource"), | 257 re.compile(r"CONSOLE MESSAGE: (line \d+: )?Not allowed to load local res
ource"), |
| 256 # Dromaeo reports values for subtests. Ignore them for now. | 258 # Dromaeo reports values for subtests. Ignore them for now. |
| 257 re.compile(r'(?P<name>.+): \[(?P<values>(\d+(.\d+)?,\s+)*\d+(.\d+)?)\]')
, | 259 re.compile(r'(?P<name>.+): \[(?P<values>(\d+(.\d+)?,\s+)*\d+(.\d+)?)\]')
, |
| 258 ] | 260 ] |
| 259 | 261 |
| 260 def _filter_output(self, output): | 262 def _filter_output(self, output): |
| 261 if output.error: | 263 if output.error: |
| 262 output.error = '\n'.join([line for line in re.split('\n', output.err
or) if not self._should_ignore_line(self._lines_to_ignore_in_stderr, line)]) | 264 output.error = '\n'.join([line for line in re.split('\n', output.err
or) |
| 265 if not self._should_ignore_line(self._line
s_to_ignore_in_stderr, line)]) |
| 263 if output.text: | 266 if output.text: |
| 264 output.text = '\n'.join([line for line in re.split('\n', output.text
) if not self._should_ignore_line(self._lines_to_ignore_in_parser_result, line)]
) | 267 output.text = '\n'.join([line for line in re.split('\n', output.text
) |
| 268 if not self._should_ignore_line(self._lines
_to_ignore_in_parser_result, line)]) |
| 265 | 269 |
| 266 | 270 |
| 267 class SingleProcessPerfTest(PerfTest): | 271 class SingleProcessPerfTest(PerfTest): |
| 272 |
| 268 def __init__(self, port, test_name, test_path, test_runner_count=1): | 273 def __init__(self, port, test_name, test_path, test_runner_count=1): |
| 269 super(SingleProcessPerfTest, self).__init__(port, test_name, test_path,
test_runner_count) | 274 super(SingleProcessPerfTest, self).__init__(port, test_name, test_path,
test_runner_count) |
| 270 | 275 |
| 271 | 276 |
| 272 class ChromiumStylePerfTest(PerfTest): | 277 class ChromiumStylePerfTest(PerfTest): |
| 273 _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+
(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$') | 278 _chromium_style_result_regex = re.compile(r'^RESULT\s+(?P<name>[^=]+)\s*=\s+
(?P<value>\d+(\.\d+)?)\s*(?P<unit>\w+)$') |
| 274 | 279 |
| 275 def __init__(self, port, test_name, test_path, test_runner_count=DEFAULT_TES
T_RUNNER_COUNT): | 280 def __init__(self, port, test_name, test_path, test_runner_count=DEFAULT_TES
T_RUNNER_COUNT): |
| 276 super(ChromiumStylePerfTest, self).__init__(port, test_name, test_path,
test_runner_count) | 281 super(ChromiumStylePerfTest, self).__init__(port, test_name, test_path,
test_runner_count) |
| 277 | 282 |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 309 (re.compile(r'^Dromaeo/'), SingleProcessPerfTest), | 314 (re.compile(r'^Dromaeo/'), SingleProcessPerfTest), |
| 310 (re.compile(r'^inspector/'), ChromiumStylePerfTest), | 315 (re.compile(r'^inspector/'), ChromiumStylePerfTest), |
| 311 ] | 316 ] |
| 312 | 317 |
| 313 @classmethod | 318 @classmethod |
| 314 def create_perf_test(cls, port, test_name, path, test_runner_count=DEFAULT_T
EST_RUNNER_COUNT): | 319 def create_perf_test(cls, port, test_name, path, test_runner_count=DEFAULT_T
EST_RUNNER_COUNT): |
| 315 for (pattern, test_class) in cls._pattern_map: | 320 for (pattern, test_class) in cls._pattern_map: |
| 316 if pattern.match(test_name): | 321 if pattern.match(test_name): |
| 317 return test_class(port, test_name, path, test_runner_count) | 322 return test_class(port, test_name, path, test_runner_count) |
| 318 return PerfTest(port, test_name, path, test_runner_count) | 323 return PerfTest(port, test_name, path, test_runner_count) |
| OLD | NEW |