[tools] add --pretty switch to run_perf.py

tools/run_perf.py

 #!/usr/bin/env python
 # Copyright 2014 the V8 project authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """
 Performance runner for d8.
 
 Call e.g. with tools/run-perf.py --arch ia32 some_suite.json
 
@@ skipping 172 matching lines @@
     self.units = units
     self.results_regexp = results_regexp
     self.stddev_regexp = stddev_regexp
     self.results = []
     self.errors = []
     self.stddev = ""
 
   def ConsumeOutput(self, stdout):
     try:
       result = re.search(self.results_regexp, stdout, re.M).group(1)
-      self.results.append(str(float(result)))
+      self.results.append(float(result))
     except ValueError:
       self.errors.append("Regexp \"%s\" returned a non-numeric for test %s."
                          % (self.results_regexp, self.name))
     except:
       self.errors.append("Regexp \"%s\" didn't match for test %s."
                          % (self.results_regexp, self.name))
 
     try:
       if self.stddev_regexp and self.stddev:
         self.errors.append("Test %s should only run once since a stddev "
                            "is provided by the test." % self.name)
       if self.stddev_regexp:
         self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
     except:
       self.errors.append("Regexp \"%s\" didn't match for test %s."
                          % (self.stddev_regexp, self.name))
 
+  def GetDeviation(self):
+    # If the benchmark provided a stddev use that directly.
+    if self.stddev:
+      return self.stddev
+    # If no stddev was provided calculate it from the results.
+    if len(self.results) == 0:
+      return 0
+    mean = self.GetMean()
+    square_deviation = sum((x-mean)**2 for x in self.results)
+    return (square_deviation / len(self.results)) ** 0.5
+
+  def GetMean(self):
+    if len(self.results) == 0:
+      return 0
+    # If the tests provided a stddev the results consists of one single average
+    # value, so return that instead.
+    if self.stddev:
+      return self.results[0]
+    # For a non-zero length results list calculate the average here.
+    return sum(self.results) / len(self.results)
+
   def GetResults(self):
     return Results([{
       "graphs": self.graphs,
       "units": self.units,
-      "results": self.results,
-      "stddev": self.stddev,
+      "results": [str(x) for x in self.results],
+      "average": self.GetMean(),
+      "stddev": self.GetDeviation(),
     }], self.errors)
 
 
 class NullMeasurement(object):
   """Null object to avoid having extra logic for configurations that didn't
   run like running without patch on trybots.
   """
   def ConsumeOutput(self, stdout):
     pass
 
@@ skipping 171 matching lines @@
     self.timeout = suite.get("timeout_%s" % arch, self.timeout)
     self.units = suite.get("units", parent.units)
     self.total = suite.get("total", parent.total)
 
     # A regular expression for results. If the parent graph provides a
     # regexp and the current suite has none, a string place holder for the
     # suite name is expected.
     # TODO(machenbach): Currently that makes only sense for the leaf level.
     # Multiple place holders for multiple levels are not supported.
     if parent.results_regexp:
-      regexp_default = parent.results_regexp % re.escape(suite["name"])
+      try:
+        regexp_default = parent.results_regexp % re.escape(suite["name"])
+      except TypeError:
+        regexp_default = parent.results_regexp
     else:
       regexp_default = None
     self.results_regexp = suite.get("results_regexp", regexp_default)
 
     # A similar regular expression for the standard deviation (optional).
     if parent.stddev_regexp:
       stddev_default = parent.stddev_regexp % re.escape(suite["name"])
     else:
       stddev_default = None
     self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
@@ skipping 160 matching lines @@
     self.shell_dir = options.shell_dir
     self.shell_dir_no_patch = options.shell_dir_no_patch
     self.extra_flags = options.extra_flags.split()
 
   @staticmethod
   def GetPlatform(options):
     if options.android_build_tools:
       return AndroidPlatform(options)
     else:
       return DesktopPlatform(options)
 

[Camillo Bruni, 2016/02/11 13:20:29]

Moved all basic "public" methods up here.

+  def GetPrettyFormatted(self, options):
+    return self
+
+  def PreExecution(self):
+    pass
+
+  def PostExecution(self):
+    pass
+
+  def PreTests(self, node, path):
+    pass
+
+  def PrintResult(self, result):
+    pass
+
   def _Run(self, runnable, count, no_patch=False):
     raise NotImplementedError()  # pragma: no cover
 
   def Run(self, runnable, count):
     """Execute the benchmark's main file.
 
     If options.shell_dir_no_patch is specified, the benchmark is run once with
     and once without patch.
     Args:
       runnable: A Runnable benchmark instance.
       count: The number of this (repeated) run.
     Returns: A tuple with the benchmark outputs with and without patch. The
              latter will be None if options.shell_dir_no_patch was not
              specified.
     """
     stdout = self._Run(runnable, count, no_patch=False)
     if self.shell_dir_no_patch:
       return stdout, self._Run(runnable, count, no_patch=True)
     else:
       return stdout, None
 
 
+class PlatformFormattedMixin(object):
+  """
+  Helper mixin that adds formatted output used when running benchmarks
+  with the --pretty flag.
+  """
+  def PrintResult(self, result):
+    if result.errors:
+      print "\r:Errors:"
+      print "\n".join(set(result.errors))
+    else:
+      trace = result.traces[0]
+      average = trace['average']
+      stdev = trace['stddev']
+      stdev_percentage = 100 * stdev / average if average != 0 else 0
+      result_string = "\r    %s +/- %3.2f%% %s" % (
+                        average, stdev_percentage, trace['units'])
+      sys.stdout.write(result_string)
+      # Fill with spaces up to 80 characters.
+      sys.stdout.write(' '*max(0, 80-len(result_string)))
+      sys.stdout.write("\n")
+      sys.stdout.flush()
+
+  def _PrintStdout(self, title, output):
+    sys.stdout.write("\r")
+    if output.exit_code != 0:
+      print output.stdout
+      return
+    # Assume the time is on the last line
+    result_line = output.stdout.splitlines()[-1].strip()
+    sys.stdout.write(result_line)
+    # Fill with spaces up to 80 characters.
+    sys.stdout.write(' '*max(0, 80-len(result_line)))
+    sys.stdout.flush()
+
+
 class DesktopPlatform(Platform):
   def __init__(self, options):
     super(DesktopPlatform, self).__init__(options)
 
-  def PreExecution(self):
-    pass
-
-  def PostExecution(self):
-    pass
+  def GetPrettyFormatted(self, options):
+    return PrettyFormattedDesktopPlatform(options)
 
   def PreTests(self, node, path):
     if isinstance(node, RunnableConfig):
       node.ChangeCWD(path)
 
+  def _PrintStdout(self, title, output):
+    print title % "Stdout"
+    print output.stdout
+
   def _Run(self, runnable, count, no_patch=False):
     suffix = ' - without patch' if no_patch else ''
     shell_dir = self.shell_dir_no_patch if no_patch else self.shell_dir
     title = ">>> %%s (#%d)%s:" % ((count + 1), suffix)
     try:
       output = commands.Execute(
           runnable.GetCommand(shell_dir, self.extra_flags),
           timeout=runnable.timeout,
       )
     except OSError as e:  # pragma: no cover
       print title % "OSError"
       print e
       return ""
-    print title % "Stdout"
-    print output.stdout
+    self._PrintStdout(title, output)
     if output.stderr:  # pragma: no cover
       # Print stderr for debugging.
       print title % "Stderr"
       print output.stderr
     if output.timed_out:
       print ">>> Test timed out after %ss." % runnable.timeout
     if '--prof' in self.extra_flags:
       os_prefix = {"linux": "linux", "macos": "mac"}.get(utils.GuessOS())
       if os_prefix:
         tick_tools = os.path.join(TOOLS_BASE, "%s-tick-processor" % os_prefix)
         subprocess.check_call(tick_tools + " --only-summary", shell=True)
       else:  # pragma: no cover
         print "Profiler option currently supported on Linux and Mac OS."
     return output.stdout
 
 
+class PrettyFormattedDesktopPlatform(PlatformFormattedMixin, DesktopPlatform):
+  pass
+
+
 class AndroidPlatform(Platform):  # pragma: no cover
   DEVICE_DIR = "/data/local/tmp/v8/"
 
   def __init__(self, options):
     super(AndroidPlatform, self).__init__(options)
     LoadAndroidBuildTools(options.android_build_tools)
 
     if not options.device:
       # Detect attached device if not specified.
       devices = adb_wrapper.AdbWrapper.Devices()
       assert devices and len(devices) == 1, (
           "None or multiple devices detected. Please specify the device on "
           "the command-line with --device")
       options.device = str(devices[0])
     self.adb_wrapper = adb_wrapper.AdbWrapper(options.device)
     self.device = device_utils.DeviceUtils(self.adb_wrapper)
 
+  def GetPrettyFormatted(self, options):
+    return PrettyFormattedAndroidPlatform(options)
+
   def PreExecution(self):
     perf = perf_control.PerfControl(self.device)
     perf.SetHighPerfMode()
 
     # Remember what we have already pushed to the device.
     self.pushed = set()
 
   def PostExecution(self):
     perf = perf_control.PerfControl(self.device)
     perf.SetDefaultPerfMode()
@@ skipping 90 matching lines @@
       bench_rel = "."
 
     try:
       output = self.device.RunShellCommand(
           cmd,
           cwd=os.path.join(AndroidPlatform.DEVICE_DIR, bench_rel),
           timeout=runnable.timeout,
           retries=0,
       )
       stdout = "\n".join(output)
       print title % "Stdout"

[Michael Achenbach, 2016/02/11 13:48:30]

You could add the self._PrintStdout(title, output) method here too and pull it
up to Platform.

       print stdout
     except device_errors.CommandTimeoutError:
       print ">>> Test timed out after %ss." % runnable.timeout
       stdout = ""
     return stdout
 
 
+class PrettyFormattedAndroidPlatform(PlatformFormattedMixin, AndroidPlatform):
+  pass
+
+
 # TODO: Implement results_processor.
 def Main(args):
   logging.getLogger().setLevel(logging.INFO)
   parser = optparse.OptionParser()
   parser.add_option("--android-build-tools",
                     help="Path to chromium's build/android. Specifying this "
                          "option will run tests using android platform.")
   parser.add_option("--arch",
                     help=("The architecture to run tests for, "
                           "'auto' or 'native' for auto-detect"),
                     default="x64")
   parser.add_option("--buildbot",
                     help="Adapt to path structure used on buildbots",
                     default=False, action="store_true")
   parser.add_option("--device",
                     help="The device ID to run Android tests on. If not given "
                          "it will be autodetected.")
   parser.add_option("--extra-flags",
                     help="Additional flags to pass to the test executable",
                     default="")
   parser.add_option("--json-test-results",
                     help="Path to a file for storing json results.")
   parser.add_option("--json-test-results-no-patch",
                     help="Path to a file for storing json results from run "
                          "without patch.")
   parser.add_option("--outdir", help="Base directory with compile output",
                     default="out")
   parser.add_option("--outdir-no-patch",
                     help="Base directory with compile output without patch")
+  parser.add_option("--pretty",
+                    help="Print human readable output",
+                    default=False, action="store_true")
   parser.add_option("--binary-override-path",
                     help="JavaScript engine binary. By default, d8 under "
                     "architecture-specific build dir. "
                     "Not supported in conjunction with outdir-no-patch.")
 
   (options, args) = parser.parse_args(args)
 
   if len(args) == 0:  # pragma: no cover
     parser.print_help()
     return 1
@@ skipping 35 matching lines @@
     options.shell_dir = os.path.dirname(options.binary_override_path)
     default_binary_name = os.path.basename(options.binary_override_path)
 
   if options.outdir_no_patch:
     options.shell_dir_no_patch = os.path.join(
         workspace, options.outdir_no_patch, build_config)
   else:
     options.shell_dir_no_patch = None
 
   platform = Platform.GetPlatform(options)
+  if options.pretty:
+    platform = platform.GetPrettyFormatted(options)
 
   results = Results()
   results_no_patch = Results()
   for path in args:
     path = os.path.abspath(path)
 
     if not os.path.exists(path):  # pragma: no cover
       results.errors.append("Configuration file %s does not exist." % path)
       continue
 
@@ skipping 21 matching lines @@
       def Runner():
         """Output generator that reruns several times."""
         for i in xrange(0, max(1, runnable.run_count)):
           # TODO(machenbach): Allow timeout per arch like with run_count per
           # arch.
           yield platform.Run(runnable, i)
 
       # Let runnable iterate over all runs and handle output.
       result, result_no_patch = runnable.Run(
           Runner, trybot=options.shell_dir_no_patch)
+      platform.PrintResult(result)
       results += result
       results_no_patch += result_no_patch
     platform.PostExecution()
 
   if options.json_test_results:
     results.WriteToFile(options.json_test_results)
   else:  # pragma: no cover
-    print results
+    if not options.pretty:
+      print results
 
   if options.json_test_results_no_patch:
     results_no_patch.WriteToFile(options.json_test_results_no_patch)
   else:  # pragma: no cover
-    print results_no_patch
+    if not options.pretty:
+      print results_no_patch
 
   return min(1, len(results.errors))
 
 if __name__ == "__main__":  # pragma: no cover
   sys.exit(Main(sys.argv[1:]))