perf-to-html.py - render JSON try perf jobs in a pleasing way.

tools/perf-to-html.py

Index: tools/perf-to-html.py
diff --git a/tools/perf-to-html.py b/tools/perf-to-html.py
new file mode 100755
index 0000000000000000000000000000000000000000..2a90202dcd5d88da6f33aa331b5ce0da9048802a
--- /dev/null
+++ b/tools/perf-to-html.py
@@ -0,0 +1,407 @@
+#!/usr/bin/env python
+# Copyright 2015 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+'''
+python %prog
+
+Convert a perf trybot JSON file into a pleasing HTML page. It can read
+from standard input or via the --filename option. Examples:
+
+  cat results.json | %prog --title "ia32 results"
+  %prog -f results.json -t "ia32 results" -o results.html
+'''
+
+from optparse import OptionParser
+import os

[Michael Achenbach, 2015/03/25 14:49:42]

nit: sort

[mvstanton, 2015/03/25 15:37:49]

Done.

+import sys
+import json
+import math
+import tempfile
+import commands
+import shutil
+
+PERCENT_CONSIDERED_SIGNIFICANT = 0.5
+PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
+PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
+
+v8_benchmarks = ["V8", "Octane", "Richards", "DeltaBlue", "Crypto",
+                 "EarleyBoyer", "RayTrace", "RegExp", "Splay", "SplayLatency",
+                 "NavierStokes", "PdfJS", "Mandreel", "MandreelLatency",
+                 "Gameboy", "CodeLoad", "Box2D", "zlib", "Typescript"]
+
+print_output = ""
+
+def Print(str_data):
+  global print_output
+  print_output += str_data
+  print_output += "\n"
+
+
+def PrintBuffer(opts):
+  global print_output
+  if opts.output <> None:

[Michael Achenbach, 2015/03/25 14:49:42]

if opts.output:
  ...

[mvstanton, 2015/03/25 15:37:49]

Done.

+    # create a file
+    text_file = open(opts.output, "w")

[Michael Achenbach, 2015/03/25 14:49:42]

with open(opts.output, "w") as text_file:
  text_file.write(print_output)

No closing necessary

[mvstanton, 2015/03/25 15:37:49]

Done.

+    text_file.write(print_output)
+    text_file.close()
+  else:
+    print(print_output)
+  print_output = ""
+
+
+def ComputeZ(baseline_avg, baseline_sigma, mean, n):
+  if baseline_sigma == 0:
+    return 1000.0;
+  return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
+
+
+# Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
+def ComputeProbability(z):
+  if z > 2.575829: # p 0.005: two sided < 0.01
+    return 0
+  if z > 2.326348: # p 0.010
+    return 0.01
+  if z > 2.170091: # p 0.015
+    return 0.02
+  if z > 2.053749: # p 0.020
+    return 0.03
+  if z > 1.959964: # p 0.025: two sided < 0.05
+    return 0.04
+  if z > 1.880793: # p 0.030
+    return 0.05
+  if z > 1.811910: # p 0.035
+    return 0.06
+  if z > 1.750686: # p 0.040
+    return 0.07
+  if z > 1.695397: # p 0.045
+    return 0.08
+  if z > 1.644853: # p 0.050: two sided < 0.10
+    return 0.09
+  if z > 1.281551: # p 0.100: two sided < 0.20
+    return 0.10
+  return 0.20 # two sided p >= 0.20
+
+
+def PercentColor(change_percent, flakyness):

[Michael Achenbach, 2015/03/25 14:49:42]

This function seems to be unused.

[mvstanton, 2015/03/25 15:37:49]

Done.

+  result = ""
+  if change_percent >= PERCENT_CONSIDERED_SIGNIFICANT:
+    result = "$GREEN"
+  elif change_percent <= -PERCENT_CONSIDERED_SIGNIFICANT:
+    result = "$RED"
+  else:

[Michael Achenbach, 2015/03/25 14:49:42]

Why return here? Isn't bold black a valid and useful result?

[mvstanton, 2015/03/25 15:37:49]

Done.

+    return ""
+  if flakyness < PROBABILITY_CONSIDERED_SIGNIFICANT:
+    result += "$BOLD"
+  elif flakyness > PROBABILITY_CONSIDERED_MEANINGLESS:
+    result = ""
+  return result
+
+
+class Result:
+  def __init__(self, test_name, count, result, sigma,
+               master_result, master_sigma):
+    self.result_ = float(result)
+    self.sigma_ = float(sigma)
+    self.master_result_ = float(master_result)
+    self.master_sigma_ = float(master_sigma)
+    self.significant_ = False
+    self.notable_ = 0
+    self.percentage_string_ = ""
+    # compute notability and significance.
+    if test_name in v8_benchmarks:

[Michael Achenbach, 2015/03/25 14:49:42]

Please make this decision based on the units property at every trace (aka line
item) in the json output. unit == "score" means bigger is better else smaller is
better.

[mvstanton, 2015/03/25 15:37:49]

Done.

+      compare_num = 100*self.result_/self.master_result_ - 100
+    else:
+      compare_num = 100*self.master_result_/self.result_ - 100
+    if abs(compare_num) > 0.1:
+      self.percentage_string_ = "%3.1f" % (compare_num)
+      z = ComputeZ(self.master_result_, self.master_sigma_, self.result_, count)
+      p = ComputeProbability(z)
+      if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
+        self.significant_ = True
+      if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
+        self.notable_ = 1
+      elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
+        self.notable_ = -1
+
+  def result(self):
+    return self.result_
+
+  def sigma(self):
+    return self.sigma_
+
+  def master_result(self):
+    return self.master_result_
+
+  def master_sigma(self):
+    return self.master_sigma_
+
+  def percentage_string(self):
+    return self.percentage_string_;
+
+  def isSignificant(self):
+    return self.significant_
+
+  def isNotablyPositive(self):
+    return self.notable_ > 0
+
+  def isNotablyNegative(self):
+    return self.notable_ < 0
+
+
+class Benchmark:
+  def __init__(self, name):
+    self.name_ = name
+    self.tests_ = {}
+
+  # tests is a dictionary of Results
+  def tests(self):
+    return self.tests_
+
+  def SortedTestKeys(self):
+    keys = self.tests_.keys()
+    keys.sort()
+    t = "Total"
+    if t in keys:
+      keys.remove(t)
+      keys.append(t)
+    return keys
+
+  def name(self):
+    return self.name_
+
+  def appendResult(self, test_name, test_data):
+    with_string = test_data["result with patch   "]
+    data = with_string.split()
+    master_string = test_data["result without patch"]
+    master_data = master_string.split()
+    runs = int(test_data["runs"])

[Michael Achenbach, 2015/03/25 14:49:42]

Here you could also access test_data["unit"].

[mvstanton, 2015/03/25 15:37:49]

Done.

+    self.tests_[test_name] = Result(test_name,
+                                    runs,
+                                    data[0], data[2],
+                                    master_data[0], master_data[2])
+
+
+def CreateBenchmark(name, data):
+  benchmark = Benchmark(name)
+  saved_data = None
+  for test in data:
+    # strip off "<name>/" prefix
+    test_name = test.split("/")[1]
+    benchmark.appendResult(test_name, data[test])
+
+  return benchmark
+
+
+def bold(data):
+  return "<b>" + data + "</b>"
+
+
+def red(data):
+  return "<font color=\"red\">" + data + "</font>"
+
+
+def green(data):
+  return "<font color=\"green\">" + data + "</font>"
+
+
+def RenderBenchmark(benchmark):
+  Print("<h2>")
+  Print("<a name=\"" + benchmark.name() + "\">")
+  Print(benchmark.name() + "</a> <a href=\"#top\">(top)</a>")
+  Print("</h2>");
+  Print("<table class=\"benchmark\">")
+  Print("<thead>")
+  Print("  <th>Test</th>")
+  Print("  <th>Result</th>")
+  Print("  <th>Master</th>")
+  Print("  <th>%</th>")
+  Print("</thead>")
+  Print("<tbody>")
+  tests = benchmark.tests()
+  for test in benchmark.SortedTestKeys():
+    t = tests[test]
+    Print("  <tr>")
+    Print("    <td>" + test + "</td>")
+    Print("    <td>" + str(t.result()) + "</td>")
+    Print("    <td>" + str(t.master_result()) + "</td>")
+    t = tests[test]
+    res = t.percentage_string()
+    if t.isSignificant():
+      res = bold(res)
+    if t.isNotablyPositive():
+      res = green(res)
+    elif t.isNotablyNegative():
+      res = red(res)
+    Print("    <td>" + res + "</td>")
+    Print("  </tr>")
+  Print("</tbody>")
+  Print("</table>")
+
+def ProcessJSONData(data, title):
+  Print("<h1>" + title + "</h1>")
+  Print("<ul>")
+  for benchmark in data:
+   if benchmark != "errors":
+     Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
+  Print("</ul>")
+  for benchmark in data:
+    if benchmark != "errors":
+      benchmark_object = CreateBenchmark(benchmark, data[benchmark])
+      RenderBenchmark(benchmark_object)
+
+
+def PrintHeader():
+  data = """<html>
+<head>
+<title>Output</title>
+<style type="text/css">
+/*
+Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
+*/
+body {
+  font-family: Helvetica, arial, sans-serif;
+  font-size: 14px;
+  line-height: 1.6;
+  padding-top: 10px;
+  padding-bottom: 10px;
+  background-color: white;
+  padding: 30px;
+}
+h1, h2, h3, h4, h5, h6 {
+  margin: 20px 0 10px;
+  padding: 0;
+  font-weight: bold;
+  -webkit-font-smoothing: antialiased;
+  cursor: text;
+  position: relative;
+}
+h1 {
+  font-size: 28px;
+  color: black;
+}
+
+h2 {
+  font-size: 24px;
+  border-bottom: 1px solid #cccccc;
+  color: black;
+}
+
+h3 {
+  font-size: 18px;
+}
+
+h4 {
+  font-size: 16px;
+}
+
+h5 {
+  font-size: 14px;
+}
+
+h6 {
+  color: #777777;
+  font-size: 14px;
+}
+
+p, blockquote, ul, ol, dl, li, table, pre {
+  margin: 15px 0;
+}
+
+li p.first {
+  display: inline-block;
+}
+
+ul, ol {
+  padding-left: 30px;
+}
+
+ul :first-child, ol :first-child {
+  margin-top: 0;
+}
+
+ul :last-child, ol :last-child {
+  margin-bottom: 0;
+}
+
+table {
+  padding: 0;
+}
+
+table tr {
+  border-top: 1px solid #cccccc;
+  background-color: white;
+  margin: 0;
+  padding: 0;
+}
+
+table tr:nth-child(2n) {
+  background-color: #f8f8f8;
+}
+
+table tr th {
+  font-weight: bold;
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px;
+}
+table tr td {
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px;
+}
+table tr th :first-child, table tr td :first-child {
+  margin-top: 0;
+}
+table tr th :last-child, table tr td :last-child {
+  margin-bottom: 0;
+}
+</style>
+</head>
+<body>
+"""
+  Print(data)
+
+
+def PrintFooter():
+  data = """</body>
+</html>
+"""
+  Print(data)
+
+
+def CreateFile(opts, args):
+  PrintHeader()
+  if opts.filename <> None:

[Michael Achenbach, 2015/03/25 14:49:43]

if opts.filename:

[mvstanton, 2015/03/25 15:37:49]

Done.

+    json_data = open(opts.filename)

[Michael Achenbach, 2015/03/25 14:49:42]

with open(opts.filename) as json_data:
  data = json.load(json_data)

[mvstanton, 2015/03/25 15:37:49]

Done.

+    data = json.load(json_data)
+    json_data.close()
+  else:
+    # load data from stdin
+    data = json.load(sys.stdin)
+
+  if opts.title <> None:

[Michael Achenbach, 2015/03/25 14:49:42]

if opts.title:

[mvstanton, 2015/03/25 15:37:49]

Done.

+    title = opts.title
+  elif opts.filename <> None:

[Michael Achenbach, 2015/03/25 14:49:42]

elif opts.filename:

[mvstanton, 2015/03/25 15:37:49]

Done.

+    title = opts.filename
+  else:
+    title = "Benchmark results"
+  ProcessJSONData(data, title)
+  PrintFooter()
+
+
+if __name__ == '__main__':
+  parser = OptionParser(usage=__doc__)
+  parser.add_option("-f", "--filename", dest="filename",
+                    help="Specifies the filename for the JSON results "\

[Michael Achenbach, 2015/03/25 14:49:42]

No need for \ - just align the strings e.g.:
parser.add_option("-f", "--filename", dest="filename",
                  help="bla"
                       "bla")

[mvstanton, 2015/03/25 15:37:49]

Done.

+"rather than reading from stdin.")
+  parser.add_option("-t", "--title", dest="title",
+                    help="Optional title of the web page.")
+  parser.add_option("-o", "--output", dest="output",
+                    help="Write html output to this file rather than stdout.")
+
+  (opts, args) = parser.parse_args()
+  CreateFile(opts, args)
+  PrintBuffer(opts)