| OLD | NEW |
| 1 ''' | 1 ''' |
| 2 Created on May 16, 2011 | 2 Created on May 16, 2011 |
| 3 | 3 |
| 4 @author: bungeman | 4 @author: bungeman |
| 5 ''' | 5 ''' |
| 6 import bench_util | 6 import bench_util |
| 7 import getopt | 7 import getopt |
| 8 import httplib | 8 import httplib |
| 9 import itertools | 9 import itertools |
| 10 import json | 10 import json |
| (...skipping 10 matching lines...) Expand all Loading... |
| 21 # Indices for getting elements from bench expectation files. | 21 # Indices for getting elements from bench expectation files. |
| 22 # See bench_expectations_<builder>.txt for details. | 22 # See bench_expectations_<builder>.txt for details. |
| 23 EXPECTED_IDX = -3 | 23 EXPECTED_IDX = -3 |
| 24 LB_IDX = -2 | 24 LB_IDX = -2 |
| 25 UB_IDX = -1 | 25 UB_IDX = -1 |
| 26 | 26 |
| 27 # Indices of the tuple of dictionaries containing slower and faster alerts. | 27 # Indices of the tuple of dictionaries containing slower and faster alerts. |
| 28 SLOWER = 0 | 28 SLOWER = 0 |
| 29 FASTER = 1 | 29 FASTER = 1 |
| 30 | 30 |
| 31 # URL prefix for the bench dashboard page. Showing recent 15 days of data. |
| 32 DASHBOARD_URL_PREFIX = 'http://go/skpdash/#15' |
| 33 |
| 31 def usage(): | 34 def usage(): |
| 32 """Prints simple usage information.""" | 35 """Prints simple usage information.""" |
| 33 | 36 |
| 34 print '-a <representation_alg> bench representation algorithm to use. ' | 37 print '-a <representation_alg> bench representation algorithm to use. ' |
| 35 print ' Defaults to "25th". See bench_util.py for details.' | 38 print ' Defaults to "25th". See bench_util.py for details.' |
| 36 print '-b <builder> name of the builder whose bench data we are checking.' | 39 print '-b <builder> name of the builder whose bench data we are checking.' |
| 37 print '-d <dir> a directory containing bench_<revision>_<scalar> files.' | 40 print '-d <dir> a directory containing bench_<revision>_<scalar> files.' |
| 38 print '-e <file> file containing expected bench builder values/ranges.' | 41 print '-e <file> file containing expected bench builder values/ranges.' |
| 39 print ' Will raise exception if actual bench values are out of range.' | 42 print ' Will raise exception if actual bench values are out of range.' |
| 40 print ' See bench_expectations_<builder>.txt for data format / examples.' | 43 print ' See bench_expectations_<builder>.txt for data format / examples.' |
| (...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 140 """ | 143 """ |
| 141 # The platform for this bot, to pass to the dashboard plot. | 144 # The platform for this bot, to pass to the dashboard plot. |
| 142 platform = key_suffix[ : key_suffix.rfind('-')] | 145 platform = key_suffix[ : key_suffix.rfind('-')] |
| 143 # Tuple of dictionaries recording exceptions that are slower and faster, | 146 # Tuple of dictionaries recording exceptions that are slower and faster, |
| 144 # respectively. Each dictionary maps off_ratio (ratio of actual to expected) | 147 # respectively. Each dictionary maps off_ratio (ratio of actual to expected) |
| 145 # to a list of corresponding exception messages. | 148 # to a list of corresponding exception messages. |
| 146 exceptions = ({}, {}) | 149 exceptions = ({}, {}) |
| 147 for line in lines: | 150 for line in lines: |
| 148 line_str = str(line) | 151 line_str = str(line) |
| 149 line_str = line_str[ : line_str.find('_{')] | 152 line_str = line_str[ : line_str.find('_{')] |
| 153 # Extracts bench and config from line_str, which is in the format |
| 154 # <bench-picture-name>.skp_<config>_ |
| 155 bench, config = line_str.strip('_').split('.skp_') |
| 150 bench_platform_key = line_str + ',' + key_suffix | 156 bench_platform_key = line_str + ',' + key_suffix |
| 151 if bench_platform_key not in expectations: | 157 if bench_platform_key not in expectations: |
| 152 continue | 158 continue |
| 153 this_bench_value = lines[line] | 159 this_bench_value = lines[line] |
| 154 this_min, this_max, this_expected = expectations[bench_platform_key] | 160 this_min, this_max, this_expected = expectations[bench_platform_key] |
| 155 if this_bench_value < this_min or this_bench_value > this_max: | 161 if this_bench_value < this_min or this_bench_value > this_max: |
| 156 off_ratio = this_bench_value / this_expected | 162 off_ratio = this_bench_value / this_expected |
| 157 exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( | 163 exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( |
| 158 bench_platform_key, this_min, this_max, this_bench_value, | 164 bench_platform_key, this_min, this_max, this_bench_value, |
| 159 this_expected, (off_ratio - 1) * 100) | 165 this_expected, (off_ratio - 1) * 100) |
| 166 exception += '\n' + '~'.join([ |
| 167 DASHBOARD_URL_PREFIX, bench, platform, config]) |
| 160 if off_ratio > 1: # Bench is slower. | 168 if off_ratio > 1: # Bench is slower. |
| 161 exceptions[SLOWER].setdefault(off_ratio, []).append(exception) | 169 exceptions[SLOWER].setdefault(off_ratio, []).append(exception) |
| 162 else: | 170 else: |
| 163 exceptions[FASTER].setdefault(off_ratio, []).append(exception) | 171 exceptions[FASTER].setdefault(off_ratio, []).append(exception) |
| 164 outputs = [] | 172 outputs = [] |
| 165 for i in [SLOWER, FASTER]: | 173 for i in [SLOWER, FASTER]: |
| 166 if exceptions[i]: | 174 if exceptions[i]: |
| 167 ratios = exceptions[i].keys() | 175 ratios = exceptions[i].keys() |
| 168 ratios.sort(reverse=True) | 176 ratios.sort(reverse=True) |
| 169 li = [] | 177 li = [] |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 227 data_points = bench_util.parse_skp_bench_data(directory, rev, rep) | 235 data_points = bench_util.parse_skp_bench_data(directory, rev, rep) |
| 228 | 236 |
| 229 bench_dict = create_bench_dict(data_points) | 237 bench_dict = create_bench_dict(data_points) |
| 230 | 238 |
| 231 if bench_expectations: | 239 if bench_expectations: |
| 232 check_expectations(bench_dict, bench_expectations, platform_and_alg) | 240 check_expectations(bench_dict, bench_expectations, platform_and_alg) |
| 233 | 241 |
| 234 | 242 |
| 235 if __name__ == "__main__": | 243 if __name__ == "__main__": |
| 236 main() | 244 main() |
| OLD | NEW |