OLD | NEW |
1 ''' | 1 ''' |
2 Created on May 16, 2011 | 2 Created on May 16, 2011 |
3 | 3 |
4 @author: bungeman | 4 @author: bungeman |
5 ''' | 5 ''' |
| 6 import bench_util |
| 7 import getopt |
| 8 import httplib |
| 9 import itertools |
| 10 import json |
| 11 import os |
| 12 import re |
6 import sys | 13 import sys |
7 import getopt | 14 import urllib |
8 import re | 15 import urllib2 |
9 import os | |
10 import bench_util | |
11 import json | |
12 import xml.sax.saxutils | 16 import xml.sax.saxutils |
13 | 17 |
14 # We throw out any measurement outside this range, and log a warning. | 18 # We throw out any measurement outside this range, and log a warning. |
15 MIN_REASONABLE_TIME = 0 | 19 MIN_REASONABLE_TIME = 0 |
16 MAX_REASONABLE_TIME = 99999 | 20 MAX_REASONABLE_TIME = 99999 |
17 | 21 |
18 # Constants for prefixes in output title used in buildbot. | 22 # Constants for prefixes in output title used in buildbot. |
19 TITLE_PREAMBLE = 'Bench_Performance_for_Skia_' | 23 TITLE_PREAMBLE = 'Bench_Performance_for_Skia_' |
20 TITLE_PREAMBLE_LENGTH = len(TITLE_PREAMBLE) | 24 TITLE_PREAMBLE_LENGTH = len(TITLE_PREAMBLE) |
21 | 25 |
| 26 # Number of data points to send to appengine at once. |
| 27 DATA_POINT_BATCHSIZE = 100 |
| 28 |
| 29 def grouper(n, iterable): |
| 30 """Groups list into list of lists for a given size. See itertools doc: |
| 31 http://docs.python.org/2/library/itertools.html#module-itertools |
| 32 """ |
| 33 args = [iter(iterable)] * n |
| 34 return [[n for n in t if n] for t in itertools.izip_longest(*args)] |
| 35 |
| 36 |
22 def usage(): | 37 def usage(): |
23 """Prints simple usage information.""" | 38 """Prints simple usage information.""" |
24 | 39 |
| 40 print '-a <url> the url to use for adding bench values to app engine app.' |
| 41 print ' Example: "https://skiadash.appspot.com/add_point".' |
| 42 print ' If not set, will skip this step.' |
25 print '-b <bench> the bench to show.' | 43 print '-b <bench> the bench to show.' |
26 print '-c <config> the config to show (GPU, 8888, 565, etc).' | 44 print '-c <config> the config to show (GPU, 8888, 565, etc).' |
27 print '-d <dir> a directory containing bench_r<revision>_<scalar> files.' | 45 print '-d <dir> a directory containing bench_r<revision>_<scalar> files.' |
28 print '-e <file> file containing expected bench values/ranges.' | 46 print '-e <file> file containing expected bench values/ranges.' |
29 print ' Will raise exception if actual bench values are out of range.' | 47 print ' Will raise exception if actual bench values are out of range.' |
30 print ' See bench_expectations.txt for data format and examples.' | 48 print ' See bench_expectations.txt for data format and examples.' |
31 print '-f <revision>[:<revision>] the revisions to use for fitting.' | 49 print '-f <revision>[:<revision>] the revisions to use for fitting.' |
32 print ' Negative <revision> is taken as offset from most recent revision.' | 50 print ' Negative <revision> is taken as offset from most recent revision.' |
33 print '-i <time> the time to ignore (w, c, g, etc).' | 51 print '-i <time> the time to ignore (w, c, g, etc).' |
34 print ' The flag is ignored when -t is set; otherwise we plot all the' | 52 print ' The flag is ignored when -t is set; otherwise we plot all the' |
(...skipping 149 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
184 On Windows, we need to generate the absolute path within Python to avoid | 202 On Windows, we need to generate the absolute path within Python to avoid |
185 the operating system's 260-character pathname limit, including chdirs.""" | 203 the operating system's 260-character pathname limit, including chdirs.""" |
186 abs_path = get_abs_path(output_path) | 204 abs_path = get_abs_path(output_path) |
187 sys.stdout = open(abs_path, 'w') | 205 sys.stdout = open(abs_path, 'w') |
188 | 206 |
189 def create_lines(revision_data_points, settings | 207 def create_lines(revision_data_points, settings |
190 , bench_of_interest, config_of_interest, time_of_interest | 208 , bench_of_interest, config_of_interest, time_of_interest |
191 , time_to_ignore): | 209 , time_to_ignore): |
192 """Convert revision data into a dictionary of line data. | 210 """Convert revision data into a dictionary of line data. |
193 | 211 |
194 ({int:[BenchDataPoints]}, {str:str}, str?, str?, str?) | 212 Args: |
195 -> {Label:[(x,y)] | [n].x <= [n+1].x}""" | 213 revision_data_points: a dictionary with integer keys (revision #) and a |
| 214 list of bench data points as values |
| 215 settings: a dictionary of setting names to value |
| 216 bench_of_interest: optional filter parameters: which bench type is of |
| 217 interest. If None, process them all. |
| 218 config_of_interest: optional filter parameters: which config type is of |
| 219 interest. If None, process them all. |
| 220 time_of_interest: optional filter parameters: which timer type is of |
| 221 interest. If None, process them all. |
| 222 time_to_ignore: optional timer type to ignore |
| 223 |
| 224 Returns: |
| 225 a dictionary of this form: |
| 226 keys = Label objects |
| 227 values = a list of (x, y) tuples sorted such that x values increase |
| 228 monotonically |
| 229 """ |
196 revisions = revision_data_points.keys() | 230 revisions = revision_data_points.keys() |
197 revisions.sort() | 231 revisions.sort() |
198 lines = {} # {Label:[(x,y)] | x[n] <= x[n+1]} | 232 lines = {} # {Label:[(x,y)] | x[n] <= x[n+1]} |
199 for revision in revisions: | 233 for revision in revisions: |
200 for point in revision_data_points[revision]: | 234 for point in revision_data_points[revision]: |
201 if (bench_of_interest is not None and | 235 if (bench_of_interest is not None and |
202 not bench_of_interest == point.bench): | 236 not bench_of_interest == point.bench): |
203 continue | 237 continue |
204 | 238 |
205 if (config_of_interest is not None and | 239 if (config_of_interest is not None and |
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
279 max_up_slope = max(max_up_slope, min_slope) | 313 max_up_slope = max(max_up_slope, min_slope) |
280 min_down_slope = min(min_down_slope, min_slope) | 314 min_down_slope = min(min_down_slope, min_slope) |
281 | 315 |
282 return (max_up_slope, min_down_slope) | 316 return (max_up_slope, min_down_slope) |
283 | 317 |
284 def main(): | 318 def main(): |
285 """Parses command line and writes output.""" | 319 """Parses command line and writes output.""" |
286 | 320 |
287 try: | 321 try: |
288 opts, _ = getopt.getopt(sys.argv[1:] | 322 opts, _ = getopt.getopt(sys.argv[1:] |
289 , "b:c:d:e:f:i:l:m:o:r:s:t:x:y:" | 323 , "a:b:c:d:e:f:i:l:m:o:r:s:t:x:y:" |
290 , "default-setting=") | 324 , "default-setting=") |
291 except getopt.GetoptError, err: | 325 except getopt.GetoptError, err: |
292 print str(err) | 326 print str(err) |
293 usage() | 327 usage() |
294 sys.exit(2) | 328 sys.exit(2) |
295 | 329 |
296 directory = None | 330 directory = None |
297 config_of_interest = None | 331 config_of_interest = None |
298 bench_of_interest = None | 332 bench_of_interest = None |
299 time_of_interest = None | 333 time_of_interest = None |
300 time_to_ignore = None | 334 time_to_ignore = None |
301 bench_expectations = {} | 335 bench_expectations = {} |
| 336 appengine_url = None # used for adding data to appengine datastore |
302 rep = None # bench representation algorithm | 337 rep = None # bench representation algorithm |
303 revision_range = '0:' | 338 revision_range = '0:' |
304 regression_range = '0:' | 339 regression_range = '0:' |
305 latest_revision = None | 340 latest_revision = None |
306 requested_height = None | 341 requested_height = None |
307 requested_width = None | 342 requested_width = None |
308 title = 'Bench graph' | 343 title = 'Bench graph' |
309 settings = {} | 344 settings = {} |
310 default_settings = {} | 345 default_settings = {} |
311 | 346 |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
363 # Skip benches without value for latest revision. | 398 # Skip benches without value for latest revision. |
364 continue | 399 continue |
365 this_min, this_max = expectations[bench_platform_key] | 400 this_min, this_max = expectations[bench_platform_key] |
366 if this_bench_value < this_min or this_bench_value > this_max: | 401 if this_bench_value < this_min or this_bench_value > this_max: |
367 exceptions.append('Bench %s value %s out of range [%s, %s].' % | 402 exceptions.append('Bench %s value %s out of range [%s, %s].' % |
368 (bench_platform_key, this_bench_value, this_min, this_max)) | 403 (bench_platform_key, this_bench_value, this_min, this_max)) |
369 if exceptions: | 404 if exceptions: |
370 raise Exception('Bench values out of range:\n' + | 405 raise Exception('Bench values out of range:\n' + |
371 '\n'.join(exceptions)) | 406 '\n'.join(exceptions)) |
372 | 407 |
| 408 def write_to_appengine(line_data_dict, url, newest_revision, bot): |
| 409 """Writes latest bench values to appengine datastore. |
| 410 line_data_dict: dictionary from create_lines. |
| 411 url: the appengine url used to send bench values to write |
| 412 newest_revision: the latest revision that this script reads |
| 413 bot: the bot platform the bench is run on |
| 414 """ |
| 415 data = [] |
| 416 for label in line_data_dict.iterkeys(): |
| 417 if not label.bench.endswith('.skp') or label.time_type: |
| 418 # filter out non-picture and non-walltime benches |
| 419 continue |
| 420 config = label.config |
| 421 rev, val = line_data_dict[label][-1] |
| 422 # This assumes that newest_revision is >= the revision of the last |
| 423 # data point we have for each line. |
| 424 if rev != newest_revision: |
| 425 continue |
| 426 data.append({'master': 'Skia', 'bot': bot, |
| 427 'test': config + '/' + label.bench.replace('.skp', ''), |
| 428 'revision': rev, 'value': val, 'error': 0}) |
| 429 for curr_data in grouper(DATA_POINT_BATCHSIZE, data): |
| 430 req = urllib2.Request(appengine_url, |
| 431 urllib.urlencode({'data': json.dumps(curr_data)})) |
| 432 try: |
| 433 urllib2.urlopen(req) |
| 434 except urllib2.HTTPError, e: |
| 435 sys.stderr.write("HTTPError for JSON data %s: %s\n" % ( |
| 436 data, e)) |
| 437 except urllib2.URLError, e: |
| 438 sys.stderr.write("URLError for JSON data %s: %s\n" % ( |
| 439 data, e)) |
| 440 except httplib.HTTPException, e: |
| 441 sys.stderr.write("HTTPException for JSON data %s: %s\n" % ( |
| 442 data, e)) |
| 443 |
373 try: | 444 try: |
374 for option, value in opts: | 445 for option, value in opts: |
375 if option == "-b": | 446 if option == "-a": |
| 447 appengine_url = value |
| 448 elif option == "-b": |
376 bench_of_interest = value | 449 bench_of_interest = value |
377 elif option == "-c": | 450 elif option == "-c": |
378 config_of_interest = value | 451 config_of_interest = value |
379 elif option == "-d": | 452 elif option == "-d": |
380 directory = value | 453 directory = value |
381 elif option == "-e": | 454 elif option == "-e": |
382 read_expectations(bench_expectations, value) | 455 read_expectations(bench_expectations, value) |
383 elif option == "-f": | 456 elif option == "-f": |
384 regression_range = value | 457 regression_range = value |
385 elif option == "-i": | 458 elif option == "-i": |
(...skipping 28 matching lines...) Expand all Loading... |
414 sys.exit(2) | 487 sys.exit(2) |
415 | 488 |
416 if time_of_interest: | 489 if time_of_interest: |
417 time_to_ignore = None | 490 time_to_ignore = None |
418 | 491 |
419 # The title flag (-l) provided in buildbot slave is in the format | 492 # The title flag (-l) provided in buildbot slave is in the format |
420 # Bench_Performance_for_Skia_<platform>, and we want to extract <platform> | 493 # Bench_Performance_for_Skia_<platform>, and we want to extract <platform> |
421 # for use in platform_and_alg to track matching benches later. If title flag | 494 # for use in platform_and_alg to track matching benches later. If title flag |
422 # is not in this format, there may be no matching benches in the file | 495 # is not in this format, there may be no matching benches in the file |
423 # provided by the expectation_file flag (-e). | 496 # provided by the expectation_file flag (-e). |
| 497 bot = title # To store the platform as bot name |
424 platform_and_alg = title | 498 platform_and_alg = title |
425 if platform_and_alg.startswith(TITLE_PREAMBLE): | 499 if platform_and_alg.startswith(TITLE_PREAMBLE): |
426 platform_and_alg = ( | 500 bot = platform_and_alg[TITLE_PREAMBLE_LENGTH:] |
427 platform_and_alg[TITLE_PREAMBLE_LENGTH:] + '-' + rep) | 501 platform_and_alg = bot + '-' + rep |
428 title += ' [representation: %s]' % rep | 502 title += ' [representation: %s]' % rep |
429 | 503 |
430 latest_revision = get_latest_revision(directory) | 504 latest_revision = get_latest_revision(directory) |
431 oldest_revision, newest_revision = parse_range(revision_range) | 505 oldest_revision, newest_revision = parse_range(revision_range) |
432 oldest_regression, newest_regression = parse_range(regression_range) | 506 oldest_regression, newest_regression = parse_range(regression_range) |
433 | 507 |
434 unfiltered_revision_data_points = parse_dir(directory | 508 unfiltered_revision_data_points = parse_dir(directory |
435 , default_settings | 509 , default_settings |
436 , oldest_revision | 510 , oldest_revision |
437 , newest_revision | 511 , newest_revision |
(...skipping 16 matching lines...) Expand all Loading... |
454 , time_of_interest | 528 , time_of_interest |
455 , time_to_ignore) | 529 , time_to_ignore) |
456 | 530 |
457 regressions = create_regressions(lines | 531 regressions = create_regressions(lines |
458 , oldest_regression | 532 , oldest_regression |
459 , newest_regression) | 533 , newest_regression) |
460 | 534 |
461 output_xhtml(lines, oldest_revision, newest_revision, ignored_revision_data_
points, | 535 output_xhtml(lines, oldest_revision, newest_revision, ignored_revision_data_
points, |
462 regressions, requested_width, requested_height, title) | 536 regressions, requested_width, requested_height, title) |
463 | 537 |
| 538 if appengine_url: |
| 539 write_to_appengine(lines, appengine_url, newest_revision, bot) |
| 540 |
464 check_expectations(lines, bench_expectations, newest_revision, | 541 check_expectations(lines, bench_expectations, newest_revision, |
465 platform_and_alg) | 542 platform_and_alg) |
466 | 543 |
467 def qa(out): | 544 def qa(out): |
468 """Stringify input and quote as an xml attribute.""" | 545 """Stringify input and quote as an xml attribute.""" |
469 return xml.sax.saxutils.quoteattr(str(out)) | 546 return xml.sax.saxutils.quoteattr(str(out)) |
470 def qe(out): | 547 def qe(out): |
471 """Stringify input and escape as xml data.""" | 548 """Stringify input and escape as xml data.""" |
472 return xml.sax.saxutils.escape(str(out)) | 549 return xml.sax.saxutils.escape(str(out)) |
473 | 550 |
(...skipping 481 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
955 print '<a id="rev_link" xlink:href="" target="_top">' | 1032 print '<a id="rev_link" xlink:href="" target="_top">' |
956 print '<text id="revision" x="0" y=%s style="' % qa(font_size*2) | 1033 print '<text id="revision" x="0" y=%s style="' % qa(font_size*2) |
957 print 'font-size: %s; ' % qe(font_size) | 1034 print 'font-size: %s; ' % qe(font_size) |
958 print 'stroke: #0000dd; text-decoration: underline; ' | 1035 print 'stroke: #0000dd; text-decoration: underline; ' |
959 print '"> </text></a>' | 1036 print '"> </text></a>' |
960 | 1037 |
961 print '</svg>' | 1038 print '</svg>' |
962 | 1039 |
963 if __name__ == "__main__": | 1040 if __name__ == "__main__": |
964 main() | 1041 main() |
OLD | NEW |