| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be found | |
| 4 # in the LICENSE file. | |
| 5 | |
| 6 """ Analyze recent bench data from graphs, and output suggested ranges. | |
| 7 | |
| 8 This script reads and parses Skia benchmark values from the xhtml files | |
| 9 generated by bench_graph_svg.py, and outputs an html file containing suggested | |
| 10 bench ranges to use in bench_expectations.txt, with analytical plots. | |
| 11 """ | |
| 12 | |
| 13 __author__ = 'bensong@google.com (Ben Chen)' | |
| 14 | |
| 15 import getopt | |
| 16 import math | |
| 17 import re | |
| 18 import sys | |
| 19 import urllib | |
| 20 from datetime import datetime | |
| 21 | |
| 22 | |
| 23 # Constants for calculating suggested bench ranges. | |
| 24 WINDOW = 5 # Moving average sliding window size. | |
| 25 # We use moving average as expected bench value, and calculate average Variance | |
| 26 # of bench from the moving average. Set range to be [X_UB * Variance above | |
| 27 # moving average, X_LB * Variance below moving average] of latest revision. | |
| 28 X_UB = 4.0 | |
| 29 X_LB = 5.0 | |
| 30 | |
| 31 # List of platforms. | |
| 32 PLATFORMS = ['GalaxyNexus_4-1_Float_Release', | |
| 33 'Mac_Float_NoDebug_32', | |
| 34 'Mac_Float_NoDebug_64', | |
| 35 'MacMiniLion_Float_NoDebug_32', | |
| 36 'MacMiniLion_Float_NoDebug_64', | |
| 37 'Nexus7_4-1_Float_Release', | |
| 38 'Shuttle_Ubuntu12_ATI5770_Float_Release_64', | |
| 39 'Shuttle_Win7_Intel_Float_Release_32', | |
| 40 'Shuttle_Win7_Intel_Float_Release_64', | |
| 41 'Xoom_4-1_Float_Release' | |
| 42 ] | |
| 43 | |
| 44 # List of bench representation algorithms. Flag "-a" is chosen from the list. | |
| 45 ALGS = ['25th', 'avg', 'med', 'min'] | |
| 46 | |
| 47 # Regular expressions for parsing bench/revision values. | |
| 48 HEIGHT_RE = 'height (\d+\.\d+) corresponds to bench value (\d+\.\d+).-->' | |
| 49 REV_RE = '<rect id="(\d+)" x="(\d+\.\d+)" y="' # Revision corresponding x. | |
| 50 LINE_RE = '<polyline id="(.*)".*points="(.*)"/>' # Bench value lines. | |
| 51 | |
| 52 # Bench graph url pattern. | |
| 53 INPUT_URL_TEMPLATE = ('http://chromium-skia-gm.commondatastorage.googleapis.com' | |
| 54 '/graph-Skia_%s-2.xhtml') | |
| 55 | |
| 56 # Output HTML elements and templates. | |
| 57 HTML_HEAD = ('<html><head><title>Skia Bench Expected Ranges</title>' | |
| 58 '<script type="text/javascript" src="https://raw.github.com/google' | |
| 59 '/skia-buildbot/master/dygraph-combined.js"></script></head><body>' | |
| 60 'Please adjust values as appropriate and update benches to monitor' | |
| 61 ' in bench/bench_expectations.txt.<br><br>') | |
| 62 HTML_SUFFIX = '</body></html>' | |
| 63 GRAPH_PREFIX = ('<br>%s<br><div id="%s" style="width:400px;height:200px"></div>' | |
| 64 '<script type="text/javascript">g%s=new Dygraph(' | |
| 65 'document.getElementById("%s"),"rev,bench,alert\\n') | |
| 66 GRAPH_SUFFIX = ('",{customBars: true,"alert":{strokeWidth:0.0,drawPoints:true,' | |
| 67 'pointSize:4,highlightCircleSize:6}});</script>') | |
| 68 | |
| 69 | |
| 70 def Usage(): | |
| 71 """Prints flag usage information.""" | |
| 72 print '-a <representation-algorithm>: defaults to "25th".' | |
| 73 print ' If set, must be one of the list element in ALGS defined above.' | |
| 74 print '-b <bench-prefix>: prefix of matching bench names to analyze.' | |
| 75 print ' Only include benchmarks whose names start with this string.' | |
| 76 print ' Cannot be empty, because there are too many benches overall.' | |
| 77 print '-o <file>: html output filename. Output to STDOUT if not set.' | |
| 78 print '-p <platform-prefix>: prefix of platform names to analyze.' | |
| 79 print ' PLATFORMS has list of matching candidates. Matches all if not set.' | |
| 80 | |
| 81 def GetBenchValues(page, bench_prefix): | |
| 82 """Returns a dict of matching bench values from the given xhtml page. | |
| 83 Args: | |
| 84 page: substring used to construct the specific bench graph URL to fetch. | |
| 85 bench_prefix: only benches starting with this string will be included. | |
| 86 | |
| 87 Returns: | |
| 88 a dict mapping benchmark name and revision combinations to bench values. | |
| 89 """ | |
| 90 height = None | |
| 91 max_bench = None | |
| 92 height_scale = None | |
| 93 revisions = [] | |
| 94 x_axes = [] # For calculating corresponding revisions. | |
| 95 val_dic = {} # dict[bench_name][revision] -> bench_value | |
| 96 | |
| 97 lines = urllib.urlopen(INPUT_URL_TEMPLATE % page).readlines() | |
| 98 for line in lines: | |
| 99 height_match = re.search(HEIGHT_RE, line) | |
| 100 if height_match: | |
| 101 height = float(height_match.group(1)) | |
| 102 max_bench = float(height_match.group(2)) | |
| 103 height_scale = max_bench / height | |
| 104 | |
| 105 rev_match = re.search(REV_RE, line) | |
| 106 if rev_match: | |
| 107 revisions.append(int(rev_match.group(1))) | |
| 108 x_axes.append(float(rev_match.group(2))) | |
| 109 | |
| 110 line_match = re.search(LINE_RE, line) | |
| 111 if not line_match: | |
| 112 continue | |
| 113 bench = line_match.group(1) | |
| 114 bench = bench[:bench.find('_{')] | |
| 115 if not bench.startswith(bench_prefix): | |
| 116 continue | |
| 117 if bench not in val_dic: | |
| 118 val_dic[bench] = {} | |
| 119 | |
| 120 vals = line_match.group(2).strip().split(' ') | |
| 121 if len(vals) < WINDOW: # Too few bench data points; skip. | |
| 122 continue | |
| 123 for val in vals: | |
| 124 x, y = [float(i) for i in val.split(',')] | |
| 125 for i in range(len(x_axes)): | |
| 126 if x <= x_axes[i]: # Found corresponding bench revision. | |
| 127 break | |
| 128 val_dic[bench][revisions[i]] = float( | |
| 129 '%.3f' % ((height - y) * height_scale)) | |
| 130 | |
| 131 return val_dic | |
| 132 | |
| 133 def CreateBenchOutput(page, bench, val_dic): | |
| 134 """Returns output for the given page and bench data in dict. | |
| 135 Args: | |
| 136 page: substring of bench graph webpage, to indicate the bench platform. | |
| 137 bench: name of the benchmark to process. | |
| 138 val_dic: dict[bench_name][revision] -> bench_value. | |
| 139 | |
| 140 Returns: | |
| 141 string of html/javascript as part of the whole script output for the bench. | |
| 142 """ | |
| 143 revs = val_dic[bench].keys() | |
| 144 revs.sort() | |
| 145 # Uses moving average to calculate expected bench variance, then sets | |
| 146 # expectations and ranges accordingly. | |
| 147 variances = [] | |
| 148 moving_avgs = [] | |
| 149 points = [] | |
| 150 for rev in revs: | |
| 151 points.append(val_dic[bench][rev]) | |
| 152 if len(points) >= WINDOW: | |
| 153 moving_avgs.append(sum(points[-WINDOW:]) / WINDOW) | |
| 154 variances.append(abs(points[-1] - moving_avgs[-1])) | |
| 155 else: # For the first WINDOW-1 points, cannot calculate moving average. | |
| 156 moving_avgs.append(points[-1]) # Uses actual value as estimates. | |
| 157 variances.append(0) | |
| 158 if len(variances) >= WINDOW: | |
| 159 for i in range(WINDOW - 1): | |
| 160 # Backfills estimated variances for the first WINDOW-1 points. | |
| 161 variances[i] = variances[WINDOW - 1] | |
| 162 | |
| 163 avg_var = sum(variances) / len(variances) | |
| 164 for val in variances: # Removes outlier variances. Only does one iter. | |
| 165 if val > min(X_LB, X_UB) * avg_var: | |
| 166 variances.remove(val) | |
| 167 avg_var = sum(variances) / len(variances) | |
| 168 | |
| 169 graph_id = '%s_%s' % (bench, page.replace('-', '_')) | |
| 170 expectations = '%s,%s,%.2f,%.2f,%.2f' % (bench, page, moving_avgs[-1], | |
| 171 moving_avgs[-1] - X_LB * avg_var, | |
| 172 moving_avgs[-1] + X_UB * avg_var) | |
| 173 out = GRAPH_PREFIX % (expectations, graph_id, graph_id, graph_id) | |
| 174 for i in range(len(revs)): | |
| 175 out += '%s,%.2f;%.2f;%.2f,' % (revs[i], moving_avgs[i] - X_LB * avg_var, | |
| 176 points[i], moving_avgs[i] + X_UB * avg_var) | |
| 177 if (points[i] > moving_avgs[i] + X_UB * avg_var or | |
| 178 points[i] < moving_avgs[i] - X_LB * avg_var): # Mark as alert point. | |
| 179 out += '%.2f;%.2f;%.2f\\n' % (points[i], points[i], points[i]) | |
| 180 else: | |
| 181 out += 'NaN;NaN;NaN\\n' | |
| 182 | |
| 183 return out | |
| 184 | |
| 185 def main(): | |
| 186 """Parses flags and outputs analysis results.""" | |
| 187 try: | |
| 188 opts, _ = getopt.getopt(sys.argv[1:], 'a:b:o:p:') | |
| 189 except getopt.GetoptError, err: | |
| 190 Usage() | |
| 191 sys.exit(2) | |
| 192 | |
| 193 alg = '25th' | |
| 194 bench_prefix = None | |
| 195 out_file = None | |
| 196 platform_prefix = '' | |
| 197 for option, value in opts: | |
| 198 if option == '-a': | |
| 199 if value not in ALGS: | |
| 200 raise Exception('Invalid flag -a (%s): must be set to one of %s.' % | |
| 201 (value, str(ALGS))) | |
| 202 alg = value | |
| 203 elif option == '-b': | |
| 204 bench_prefix = value | |
| 205 elif option == '-o': | |
| 206 out_file = value | |
| 207 elif option == '-p': | |
| 208 platform_prefix = value | |
| 209 else: | |
| 210 Usage() | |
| 211 raise Exception('Error handling flags.') | |
| 212 | |
| 213 if not bench_prefix: | |
| 214 raise Exception('Must provide nonempty Flag -b (bench name prefix).') | |
| 215 | |
| 216 pages = [] | |
| 217 for platform in PLATFORMS: | |
| 218 if not platform.startswith(platform_prefix): | |
| 219 continue | |
| 220 pages.append('%s-%s' % (platform, alg)) | |
| 221 | |
| 222 if not pages: # No matching platform found. | |
| 223 raise Exception('Flag -p (platform prefix: %s) does not match any of %s.' % | |
| 224 (platform_prefix, str(PLATFORMS))) | |
| 225 | |
| 226 body = '' | |
| 227 # Iterates through bench graph xhtml pages for oututting matching benches. | |
| 228 for page in pages: | |
| 229 bench_value_dict = GetBenchValues(page, bench_prefix) | |
| 230 for bench in bench_value_dict: | |
| 231 body += CreateBenchOutput(page, bench, bench_value_dict) + GRAPH_SUFFIX | |
| 232 | |
| 233 if not body: | |
| 234 raise Exception('No bench outputs. Most likely there are no matching bench' | |
| 235 ' prefix (%s) in Flags -b for platforms %s.\nPlease also ' | |
| 236 'check if the bench graph URLs are valid at %s.' % ( | |
| 237 bench_prefix, str(PLATFORMS), INPUT_URL_TEMPLATE)) | |
| 238 if out_file: | |
| 239 f = open(out_file, 'w+') | |
| 240 f.write(HTML_HEAD + body + HTML_SUFFIX) | |
| 241 f.close() | |
| 242 else: | |
| 243 print HTML_HEAD + body + HTML_SUFFIX | |
| 244 | |
| 245 | |
| 246 if '__main__' == __name__: | |
| 247 main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 199143004:
Removes bench expectations files. New place is skia-autogen svn repo. See https://codereview.chromi… (Closed)
Base URL: https://skia.googlesource.com/skia.git@master