OLD | NEW |
1 ''' | 1 ''' |
2 Created on May 16, 2011 | 2 Created on May 16, 2011 |
3 | 3 |
4 @author: bungeman | 4 @author: bungeman |
5 ''' | 5 ''' |
6 import bench_util | 6 import bench_util |
7 import getopt | 7 import getopt |
8 import httplib | 8 import httplib |
9 import itertools | 9 import itertools |
10 import json | 10 import json |
(...skipping 10 matching lines...) Expand all Loading... |
21 # Indices for getting elements from bench expectation files. | 21 # Indices for getting elements from bench expectation files. |
22 # See bench_expectations_<builder>.txt for details. | 22 # See bench_expectations_<builder>.txt for details. |
23 EXPECTED_IDX = -3 | 23 EXPECTED_IDX = -3 |
24 LB_IDX = -2 | 24 LB_IDX = -2 |
25 UB_IDX = -1 | 25 UB_IDX = -1 |
26 | 26 |
27 # Indices of the tuple of dictionaries containing slower and faster alerts. | 27 # Indices of the tuple of dictionaries containing slower and faster alerts. |
28 SLOWER = 0 | 28 SLOWER = 0 |
29 FASTER = 1 | 29 FASTER = 1 |
30 | 30 |
| 31 # URL prefix for the bench dashboard page. Showing recent 15 days of data. |
| 32 DASHBOARD_URL_PREFIX = 'http://go/skpdash/#15' |
| 33 |
31 def usage(): | 34 def usage(): |
32 """Prints simple usage information.""" | 35 """Prints simple usage information.""" |
33 | 36 |
34 print '-a <representation_alg> bench representation algorithm to use. ' | 37 print '-a <representation_alg> bench representation algorithm to use. ' |
35 print ' Defaults to "25th". See bench_util.py for details.' | 38 print ' Defaults to "25th". See bench_util.py for details.' |
36 print '-b <builder> name of the builder whose bench data we are checking.' | 39 print '-b <builder> name of the builder whose bench data we are checking.' |
37 print '-d <dir> a directory containing bench_<revision>_<scalar> files.' | 40 print '-d <dir> a directory containing bench_<revision>_<scalar> files.' |
38 print '-e <file> file containing expected bench builder values/ranges.' | 41 print '-e <file> file containing expected bench builder values/ranges.' |
39 print ' Will raise exception if actual bench values are out of range.' | 42 print ' Will raise exception if actual bench values are out of range.' |
40 print ' See bench_expectations_<builder>.txt for data format / examples.' | 43 print ' See bench_expectations_<builder>.txt for data format / examples.' |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
140 """ | 143 """ |
141 # The platform for this bot, to pass to the dashboard plot. | 144 # The platform for this bot, to pass to the dashboard plot. |
142 platform = key_suffix[ : key_suffix.rfind('-')] | 145 platform = key_suffix[ : key_suffix.rfind('-')] |
143 # Tuple of dictionaries recording exceptions that are slower and faster, | 146 # Tuple of dictionaries recording exceptions that are slower and faster, |
144 # respectively. Each dictionary maps off_ratio (ratio of actual to expected) | 147 # respectively. Each dictionary maps off_ratio (ratio of actual to expected) |
145 # to a list of corresponding exception messages. | 148 # to a list of corresponding exception messages. |
146 exceptions = ({}, {}) | 149 exceptions = ({}, {}) |
147 for line in lines: | 150 for line in lines: |
148 line_str = str(line) | 151 line_str = str(line) |
149 line_str = line_str[ : line_str.find('_{')] | 152 line_str = line_str[ : line_str.find('_{')] |
| 153 # Extracts bench and config from line_str, which is in the format |
| 154 # <bench-picture-name>.skp_<config>_ |
| 155 bench, config = line_str.strip('_').split('.skp_') |
150 bench_platform_key = line_str + ',' + key_suffix | 156 bench_platform_key = line_str + ',' + key_suffix |
151 if bench_platform_key not in expectations: | 157 if bench_platform_key not in expectations: |
152 continue | 158 continue |
153 this_bench_value = lines[line] | 159 this_bench_value = lines[line] |
154 this_min, this_max, this_expected = expectations[bench_platform_key] | 160 this_min, this_max, this_expected = expectations[bench_platform_key] |
155 if this_bench_value < this_min or this_bench_value > this_max: | 161 if this_bench_value < this_min or this_bench_value > this_max: |
156 off_ratio = this_bench_value / this_expected | 162 off_ratio = this_bench_value / this_expected |
157 exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( | 163 exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( |
158 bench_platform_key, this_min, this_max, this_bench_value, | 164 bench_platform_key, this_min, this_max, this_bench_value, |
159 this_expected, (off_ratio - 1) * 100) | 165 this_expected, (off_ratio - 1) * 100) |
| 166 exception += '\n' + '~'.join([ |
| 167 DASHBOARD_URL_PREFIX, bench, platform, config]) |
160 if off_ratio > 1: # Bench is slower. | 168 if off_ratio > 1: # Bench is slower. |
161 exceptions[SLOWER].setdefault(off_ratio, []).append(exception) | 169 exceptions[SLOWER].setdefault(off_ratio, []).append(exception) |
162 else: | 170 else: |
163 exceptions[FASTER].setdefault(off_ratio, []).append(exception) | 171 exceptions[FASTER].setdefault(off_ratio, []).append(exception) |
164 outputs = [] | 172 outputs = [] |
165 for i in [SLOWER, FASTER]: | 173 for i in [SLOWER, FASTER]: |
166 if exceptions[i]: | 174 if exceptions[i]: |
167 ratios = exceptions[i].keys() | 175 ratios = exceptions[i].keys() |
168 ratios.sort(reverse=True) | 176 ratios.sort(reverse=True) |
169 li = [] | 177 li = [] |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
227 data_points = bench_util.parse_skp_bench_data(directory, rev, rep) | 235 data_points = bench_util.parse_skp_bench_data(directory, rev, rep) |
228 | 236 |
229 bench_dict = create_bench_dict(data_points) | 237 bench_dict = create_bench_dict(data_points) |
230 | 238 |
231 if bench_expectations: | 239 if bench_expectations: |
232 check_expectations(bench_dict, bench_expectations, platform_and_alg) | 240 check_expectations(bench_dict, bench_expectations, platform_and_alg) |
233 | 241 |
234 | 242 |
235 if __name__ == "__main__": | 243 if __name__ == "__main__": |
236 main() | 244 main() |
OLD | NEW |