OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright 2015 the V8 project authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 ''' |
| 6 python %prog |
| 7 |
| 8 Convert a perf trybot JSON file into a pleasing HTML page. It can read |
| 9 from standard input or via the --filename option. Examples: |
| 10 |
| 11 cat results.json | %prog --title "ia32 results" |
| 12 %prog -f results.json -t "ia32 results" -o results.html |
| 13 ''' |
| 14 |
| 15 import commands |
| 16 import json |
| 17 import math |
| 18 from optparse import OptionParser |
| 19 import os |
| 20 import shutil |
| 21 import sys |
| 22 import tempfile |
| 23 |
| 24 PERCENT_CONSIDERED_SIGNIFICANT = 0.5 |
| 25 PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02 |
| 26 PROBABILITY_CONSIDERED_MEANINGLESS = 0.05 |
| 27 |
| 28 |
| 29 def ComputeZ(baseline_avg, baseline_sigma, mean, n): |
| 30 if baseline_sigma == 0: |
| 31 return 1000.0; |
| 32 return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n))) |
| 33 |
| 34 |
| 35 # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html |
| 36 def ComputeProbability(z): |
| 37 if z > 2.575829: # p 0.005: two sided < 0.01 |
| 38 return 0 |
| 39 if z > 2.326348: # p 0.010 |
| 40 return 0.01 |
| 41 if z > 2.170091: # p 0.015 |
| 42 return 0.02 |
| 43 if z > 2.053749: # p 0.020 |
| 44 return 0.03 |
| 45 if z > 1.959964: # p 0.025: two sided < 0.05 |
| 46 return 0.04 |
| 47 if z > 1.880793: # p 0.030 |
| 48 return 0.05 |
| 49 if z > 1.811910: # p 0.035 |
| 50 return 0.06 |
| 51 if z > 1.750686: # p 0.040 |
| 52 return 0.07 |
| 53 if z > 1.695397: # p 0.045 |
| 54 return 0.08 |
| 55 if z > 1.644853: # p 0.050: two sided < 0.10 |
| 56 return 0.09 |
| 57 if z > 1.281551: # p 0.100: two sided < 0.20 |
| 58 return 0.10 |
| 59 return 0.20 # two sided p >= 0.20 |
| 60 |
| 61 |
| 62 class Result: |
| 63 def __init__(self, test_name, count, hasScoreUnits, result, sigma, |
| 64 master_result, master_sigma): |
| 65 self.result_ = float(result) |
| 66 self.sigma_ = float(sigma) |
| 67 self.master_result_ = float(master_result) |
| 68 self.master_sigma_ = float(master_sigma) |
| 69 self.significant_ = False |
| 70 self.notable_ = 0 |
| 71 self.percentage_string_ = "" |
| 72 # compute notability and significance. |
| 73 if hasScoreUnits: |
| 74 compare_num = 100*self.result_/self.master_result_ - 100 |
| 75 else: |
| 76 compare_num = 100*self.master_result_/self.result_ - 100 |
| 77 if abs(compare_num) > 0.1: |
| 78 self.percentage_string_ = "%3.1f" % (compare_num) |
| 79 z = ComputeZ(self.master_result_, self.master_sigma_, self.result_, count) |
| 80 p = ComputeProbability(z) |
| 81 if p < PROBABILITY_CONSIDERED_SIGNIFICANT: |
| 82 self.significant_ = True |
| 83 if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT: |
| 84 self.notable_ = 1 |
| 85 elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT: |
| 86 self.notable_ = -1 |
| 87 |
| 88 def result(self): |
| 89 return self.result_ |
| 90 |
| 91 def sigma(self): |
| 92 return self.sigma_ |
| 93 |
| 94 def master_result(self): |
| 95 return self.master_result_ |
| 96 |
| 97 def master_sigma(self): |
| 98 return self.master_sigma_ |
| 99 |
| 100 def percentage_string(self): |
| 101 return self.percentage_string_; |
| 102 |
| 103 def isSignificant(self): |
| 104 return self.significant_ |
| 105 |
| 106 def isNotablyPositive(self): |
| 107 return self.notable_ > 0 |
| 108 |
| 109 def isNotablyNegative(self): |
| 110 return self.notable_ < 0 |
| 111 |
| 112 |
| 113 class Benchmark: |
| 114 def __init__(self, name, data): |
| 115 self.name_ = name |
| 116 self.tests_ = {} |
| 117 for test in data: |
| 118 # strip off "<name>/" prefix |
| 119 test_name = test.split("/")[1] |
| 120 self.appendResult(test_name, data[test]) |
| 121 |
| 122 # tests is a dictionary of Results |
| 123 def tests(self): |
| 124 return self.tests_ |
| 125 |
| 126 def SortedTestKeys(self): |
| 127 keys = self.tests_.keys() |
| 128 keys.sort() |
| 129 t = "Total" |
| 130 if t in keys: |
| 131 keys.remove(t) |
| 132 keys.append(t) |
| 133 return keys |
| 134 |
| 135 def name(self): |
| 136 return self.name_ |
| 137 |
| 138 def appendResult(self, test_name, test_data): |
| 139 with_string = test_data["result with patch "] |
| 140 data = with_string.split() |
| 141 master_string = test_data["result without patch"] |
| 142 master_data = master_string.split() |
| 143 runs = int(test_data["runs"]) |
| 144 units = test_data["units"] |
| 145 hasScoreUnits = units == "score" |
| 146 self.tests_[test_name] = Result(test_name, |
| 147 runs, |
| 148 hasScoreUnits, |
| 149 data[0], data[2], |
| 150 master_data[0], master_data[2]) |
| 151 |
| 152 |
| 153 class BenchmarkRenderer: |
| 154 def __init__(self, output_file): |
| 155 self.print_output_ = [] |
| 156 self.output_file_ = output_file |
| 157 |
| 158 def Print(self, str_data): |
| 159 self.print_output_.append(str_data) |
| 160 |
| 161 def FlushOutput(self): |
| 162 string_data = "\n".join(self.print_output_) |
| 163 print_output = [] |
| 164 if self.output_file_: |
| 165 # create a file |
| 166 with open(self.output_file_, "w") as text_file: |
| 167 text_file.write(string_data) |
| 168 else: |
| 169 print(string_data) |
| 170 |
| 171 def RenderOneBenchmark(self, benchmark): |
| 172 self.Print("<h2>") |
| 173 self.Print("<a name=\"" + benchmark.name() + "\">") |
| 174 self.Print(benchmark.name() + "</a> <a href=\"#top\">(top)</a>") |
| 175 self.Print("</h2>"); |
| 176 self.Print("<table class=\"benchmark\">") |
| 177 self.Print("<thead>") |
| 178 self.Print(" <th>Test</th>") |
| 179 self.Print(" <th>Result</th>") |
| 180 self.Print(" <th>Master</th>") |
| 181 self.Print(" <th>%</th>") |
| 182 self.Print("</thead>") |
| 183 self.Print("<tbody>") |
| 184 tests = benchmark.tests() |
| 185 for test in benchmark.SortedTestKeys(): |
| 186 t = tests[test] |
| 187 self.Print(" <tr>") |
| 188 self.Print(" <td>" + test + "</td>") |
| 189 self.Print(" <td>" + str(t.result()) + "</td>") |
| 190 self.Print(" <td>" + str(t.master_result()) + "</td>") |
| 191 t = tests[test] |
| 192 res = t.percentage_string() |
| 193 if t.isSignificant(): |
| 194 res = self.bold(res) |
| 195 if t.isNotablyPositive(): |
| 196 res = self.green(res) |
| 197 elif t.isNotablyNegative(): |
| 198 res = self.red(res) |
| 199 self.Print(" <td>" + res + "</td>") |
| 200 self.Print(" </tr>") |
| 201 self.Print("</tbody>") |
| 202 self.Print("</table>") |
| 203 |
| 204 def ProcessJSONData(self, data, title): |
| 205 self.Print("<h1>" + title + "</h1>") |
| 206 self.Print("<ul>") |
| 207 for benchmark in data: |
| 208 if benchmark != "errors": |
| 209 self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>
") |
| 210 self.Print("</ul>") |
| 211 for benchmark in data: |
| 212 if benchmark != "errors": |
| 213 benchmark_object = Benchmark(benchmark, data[benchmark]) |
| 214 self.RenderOneBenchmark(benchmark_object) |
| 215 |
| 216 def bold(self, data): |
| 217 return "<b>" + data + "</b>" |
| 218 |
| 219 def red(self, data): |
| 220 return "<font color=\"red\">" + data + "</font>" |
| 221 |
| 222 |
| 223 def green(self, data): |
| 224 return "<font color=\"green\">" + data + "</font>" |
| 225 |
| 226 def PrintHeader(self): |
| 227 data = """<html> |
| 228 <head> |
| 229 <title>Output</title> |
| 230 <style type="text/css"> |
| 231 /* |
| 232 Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919 |
| 233 */ |
| 234 body { |
| 235 font-family: Helvetica, arial, sans-serif; |
| 236 font-size: 14px; |
| 237 line-height: 1.6; |
| 238 padding-top: 10px; |
| 239 padding-bottom: 10px; |
| 240 background-color: white; |
| 241 padding: 30px; |
| 242 } |
| 243 h1, h2, h3, h4, h5, h6 { |
| 244 margin: 20px 0 10px; |
| 245 padding: 0; |
| 246 font-weight: bold; |
| 247 -webkit-font-smoothing: antialiased; |
| 248 cursor: text; |
| 249 position: relative; |
| 250 } |
| 251 h1 { |
| 252 font-size: 28px; |
| 253 color: black; |
| 254 } |
| 255 |
| 256 h2 { |
| 257 font-size: 24px; |
| 258 border-bottom: 1px solid #cccccc; |
| 259 color: black; |
| 260 } |
| 261 |
| 262 h3 { |
| 263 font-size: 18px; |
| 264 } |
| 265 |
| 266 h4 { |
| 267 font-size: 16px; |
| 268 } |
| 269 |
| 270 h5 { |
| 271 font-size: 14px; |
| 272 } |
| 273 |
| 274 h6 { |
| 275 color: #777777; |
| 276 font-size: 14px; |
| 277 } |
| 278 |
| 279 p, blockquote, ul, ol, dl, li, table, pre { |
| 280 margin: 15px 0; |
| 281 } |
| 282 |
| 283 li p.first { |
| 284 display: inline-block; |
| 285 } |
| 286 |
| 287 ul, ol { |
| 288 padding-left: 30px; |
| 289 } |
| 290 |
| 291 ul :first-child, ol :first-child { |
| 292 margin-top: 0; |
| 293 } |
| 294 |
| 295 ul :last-child, ol :last-child { |
| 296 margin-bottom: 0; |
| 297 } |
| 298 |
| 299 table { |
| 300 padding: 0; |
| 301 } |
| 302 |
| 303 table tr { |
| 304 border-top: 1px solid #cccccc; |
| 305 background-color: white; |
| 306 margin: 0; |
| 307 padding: 0; |
| 308 } |
| 309 |
| 310 table tr:nth-child(2n) { |
| 311 background-color: #f8f8f8; |
| 312 } |
| 313 |
| 314 table tr th { |
| 315 font-weight: bold; |
| 316 border: 1px solid #cccccc; |
| 317 text-align: left; |
| 318 margin: 0; |
| 319 padding: 6px 13px; |
| 320 } |
| 321 table tr td { |
| 322 border: 1px solid #cccccc; |
| 323 text-align: left; |
| 324 margin: 0; |
| 325 padding: 6px 13px; |
| 326 } |
| 327 table tr th :first-child, table tr td :first-child { |
| 328 margin-top: 0; |
| 329 } |
| 330 table tr th :last-child, table tr td :last-child { |
| 331 margin-bottom: 0; |
| 332 } |
| 333 </style> |
| 334 </head> |
| 335 <body> |
| 336 """ |
| 337 self.Print(data) |
| 338 |
| 339 def PrintFooter(self): |
| 340 data = """</body> |
| 341 </html> |
| 342 """ |
| 343 self.Print(data) |
| 344 |
| 345 |
| 346 def Render(opts, args): |
| 347 if opts.filename: |
| 348 with open(opts.filename) as json_data: |
| 349 data = json.load(json_data) |
| 350 else: |
| 351 # load data from stdin |
| 352 data = json.load(sys.stdin) |
| 353 |
| 354 if opts.title: |
| 355 title = opts.title |
| 356 elif opts.filename: |
| 357 title = opts.filename |
| 358 else: |
| 359 title = "Benchmark results" |
| 360 renderer = BenchmarkRenderer(opts.output) |
| 361 renderer.PrintHeader() |
| 362 renderer.ProcessJSONData(data, title) |
| 363 renderer.PrintFooter() |
| 364 renderer.FlushOutput() |
| 365 |
| 366 |
| 367 if __name__ == '__main__': |
| 368 parser = OptionParser(usage=__doc__) |
| 369 parser.add_option("-f", "--filename", dest="filename", |
| 370 help="Specifies the filename for the JSON results " |
| 371 "rather than reading from stdin.") |
| 372 parser.add_option("-t", "--title", dest="title", |
| 373 help="Optional title of the web page.") |
| 374 parser.add_option("-o", "--output", dest="output", |
| 375 help="Write html output to this file rather than stdout.") |
| 376 |
| 377 (opts, args) = parser.parse_args() |
| 378 Render(opts, args) |
OLD | NEW |