OLD | NEW |
1 ''' | 1 ''' |
2 Created on May 16, 2011 | 2 Created on May 16, 2011 |
3 | 3 |
4 @author: bungeman | 4 @author: bungeman |
5 ''' | 5 ''' |
6 import bench_util | 6 import bench_util |
7 import getopt | 7 import getopt |
8 import httplib | 8 import httplib |
9 import itertools | 9 import itertools |
10 import json | 10 import json |
11 import os | 11 import os |
12 import re | 12 import re |
13 import sys | 13 import sys |
14 import urllib | 14 import urllib |
15 import urllib2 | 15 import urllib2 |
16 import xml.sax.saxutils | 16 import xml.sax.saxutils |
17 | 17 |
18 # Maximum expected number of characters we expect in an svn revision. | 18 # Maximum expected number of characters we expect in an svn revision. |
19 MAX_SVN_REV_LENGTH = 5 | 19 MAX_SVN_REV_LENGTH = 5 |
20 | 20 |
| 21 # Indices for getting elements from bench expectation files. |
| 22 # See bench_expectations_<builder>.txt for details. |
| 23 EXPECTED_IDX = -3 |
| 24 LB_IDX = -2 |
| 25 UB_IDX = -1 |
| 26 |
| 27 # Indices of the tuple of dictionaries containing slower and faster alerts. |
| 28 SLOWER = 0 |
| 29 FASTER = 1 |
| 30 |
21 def usage(): | 31 def usage(): |
22 """Prints simple usage information.""" | 32 """Prints simple usage information.""" |
23 | 33 |
24 print '-a <representation_alg> bench representation algorithm to use. ' | 34 print '-a <representation_alg> bench representation algorithm to use. ' |
25 print ' Defaults to "25th". See bench_util.py for details.' | 35 print ' Defaults to "25th". See bench_util.py for details.' |
26 print '-b <builder> name of the builder whose bench data we are checking.' | 36 print '-b <builder> name of the builder whose bench data we are checking.' |
27 print '-d <dir> a directory containing bench_<revision>_<scalar> files.' | 37 print '-d <dir> a directory containing bench_<revision>_<scalar> files.' |
28 print '-e <file> file containing expected bench builder values/ranges.' | 38 print '-e <file> file containing expected bench builder values/ranges.' |
29 print ' Will raise exception if actual bench values are out of range.' | 39 print ' Will raise exception if actual bench values are out of range.' |
30 print ' See bench_expectations_<builder>.txt for data format / examples.' | 40 print ' See bench_expectations_<builder>.txt for data format / examples.' |
(...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
127 elements = expectation.strip().split(',') | 137 elements = expectation.strip().split(',') |
128 if not elements[0] or elements[0].startswith('#'): | 138 if not elements[0] or elements[0].startswith('#'): |
129 continue | 139 continue |
130 if len(elements) != 5: | 140 if len(elements) != 5: |
131 raise Exception("Invalid expectation line format: %s" % | 141 raise Exception("Invalid expectation line format: %s" % |
132 expectation) | 142 expectation) |
133 bench_entry = elements[0] + ',' + elements[1] | 143 bench_entry = elements[0] + ',' + elements[1] |
134 if bench_entry in expectations: | 144 if bench_entry in expectations: |
135 raise Exception("Dup entries for bench expectation %s" % | 145 raise Exception("Dup entries for bench expectation %s" % |
136 bench_entry) | 146 bench_entry) |
137 # [<Bench_BmpConfig_TimeType>,<Platform-Alg>] -> (LB, UB) | 147 # [<Bench_BmpConfig_TimeType>,<Platform-Alg>] -> (LB, UB, EXPECTED) |
138 expectations[bench_entry] = (float(elements[-2]), | 148 expectations[bench_entry] = (float(elements[LB_IDX]), |
139 float(elements[-1])) | 149 float(elements[UB_IDX]), |
| 150 float(elements[EXPECTED_IDX])) |
140 | 151 |
141 def check_expectations(lines, expectations, revision, key_suffix): | 152 def check_expectations(lines, expectations, key_suffix): |
142 """Check if there are benches in the given revising out of range. | 153 """Check if any bench results are outside of expected range. |
| 154 |
| 155 For each input line in lines, checks the expectations dictionary to see if |
| 156 the bench is out of the given range. |
| 157 |
| 158 Args: |
| 159 lines: dictionary mapping Label objects to the bench values. |
| 160 expectations: dictionary returned by read_expectations(). |
| 161 key_suffix: string of <Platform>-<Alg> containing the bot platform and the |
| 162 bench representation algorithm. |
| 163 |
| 164 Returns: |
| 165 No return value. |
| 166 |
| 167 Raises: |
| 168 Exception containing bench data that are out of range, if any. |
143 """ | 169 """ |
144 # The platform for this bot, to pass to the dashboard plot. | 170 # The platform for this bot, to pass to the dashboard plot. |
145 platform = key_suffix[ : key_suffix.rfind('-')] | 171 platform = key_suffix[ : key_suffix.rfind('-')] |
146 exceptions = [] | 172 # Tuple of dictionaries recording exceptions that are slower and faster, |
| 173 # respectively. Each dictionary maps off_ratio (ratio of actual to expected) |
| 174 # to a list of corresponding exception messages. |
| 175 exceptions = ({}, {}) |
147 for line in lines: | 176 for line in lines: |
148 line_str = str(line) | 177 line_str = str(line) |
149 line_str = line_str[ : line_str.find('_{')] | 178 line_str = line_str[ : line_str.find('_{')] |
150 bench_platform_key = line_str + ',' + key_suffix | 179 bench_platform_key = line_str + ',' + key_suffix |
151 if bench_platform_key not in expectations: | 180 if bench_platform_key not in expectations: |
152 continue | 181 continue |
153 this_bench_value = lines[line] | 182 this_bench_value = lines[line] |
154 this_min, this_max = expectations[bench_platform_key] | 183 this_min, this_max, this_expected = expectations[bench_platform_key] |
155 if this_bench_value < this_min or this_bench_value > this_max: | 184 if this_bench_value < this_min or this_bench_value > this_max: |
156 exception = 'Bench %s value %s out of range [%s, %s].' % ( | 185 off_ratio = this_bench_value / this_expected |
157 bench_platform_key, this_bench_value, this_min, this_max) | 186 exception = 'Bench %s out of range [%s, %s] (%s vs %s, %s%%).' % ( |
158 exceptions.append(exception) | 187 bench_platform_key, this_min, this_max, this_bench_value, |
159 if exceptions: | 188 this_expected, (off_ratio - 1) * 100) |
160 raise Exception('Bench values out of range:\n' + | 189 if off_ratio > 1: # Bench is slower. |
161 '\n'.join(exceptions)) | 190 exceptions[SLOWER].setdefault(off_ratio, []).append(exception) |
| 191 else: |
| 192 exceptions[FASTER].setdefault(off_ratio, []).append(exception) |
| 193 outputs = [] |
| 194 for i in [SLOWER, FASTER]: |
| 195 if exceptions[i]: |
| 196 ratios = exceptions[i].keys() |
| 197 ratios.sort(reverse=True) |
| 198 li = [] |
| 199 for ratio in ratios: |
| 200 li.extend(exceptions[i][ratio]) |
| 201 header = '%s benches got slower (sorted by %% difference):' % len(li) |
| 202 if i == FASTER: |
| 203 header = header.replace('slower', 'faster') |
| 204 outputs.extend(['', header] + li) |
| 205 |
| 206 if outputs: |
| 207 raise Exception('\n'.join(outputs)) |
162 | 208 |
163 def main(): | 209 def main(): |
164 """Parses command line and checks bench expectations.""" | 210 """Parses command line and checks bench expectations.""" |
165 try: | 211 try: |
166 opts, _ = getopt.getopt(sys.argv[1:], | 212 opts, _ = getopt.getopt(sys.argv[1:], |
167 "a:b:d:e:r:", | 213 "a:b:d:e:r:", |
168 "default-setting=") | 214 "default-setting=") |
169 except getopt.GetoptError, err: | 215 except getopt.GetoptError, err: |
170 print str(err) | 216 print str(err) |
171 usage() | 217 usage() |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
203 platform_and_alg = bot + '-' + rep | 249 platform_and_alg = bot + '-' + rep |
204 | 250 |
205 data_points = parse_dir(directory, | 251 data_points = parse_dir(directory, |
206 {}, # Sets default settings to empty. | 252 {}, # Sets default settings to empty. |
207 rev, | 253 rev, |
208 rep) | 254 rep) |
209 | 255 |
210 bench_dict = create_bench_dict(data_points) | 256 bench_dict = create_bench_dict(data_points) |
211 | 257 |
212 if bench_expectations: | 258 if bench_expectations: |
213 check_expectations(bench_dict, bench_expectations, rev, | 259 check_expectations(bench_dict, bench_expectations, platform_and_alg) |
214 platform_and_alg) | |
215 | 260 |
216 | 261 |
217 if __name__ == "__main__": | 262 if __name__ == "__main__": |
218 main() | 263 main() |
OLD | NEW |