Add hardware monitoring to skpbench

tools/skpbench/skpbench.py

 #!/usr/bin/env python
 
 # Copyright 2016 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from __future__ import print_function
+from _adb import Adb
 from _benchresult import BenchResult
+from _hardware import HardwareException, Hardware
 from argparse import ArgumentParser
 from queue import Queue
-from threading import Thread
+from threading import Thread, Timer
 import collections
 import glob
 import math
 import re
 import subprocess
 import sys
+import time
 
-__argparse = ArgumentParser(description="""
+__argparse = ArgumentParser(description='''
 
 Executes the skpbench binary with various configs and skps.
 
 Also monitors the output in order to filter out and re-run results that have an
 unacceptable stddev.
 
-""")
+''')
 
 __argparse.add_argument('--adb',
-    action='store_true', help='execute skpbench over adb')
+    action='store_true', help="execute skpbench over adb")
 __argparse.add_argument('-s', '--device-serial',
-    help='if using adb, id of the specific device to target')
+    help="if using adb, id of the specific device to target")
 __argparse.add_argument('-p', '--path',
-    help='directory to execute ./skpbench from')
+    help="directory to execute ./skpbench from")
 __argparse.add_argument('-m', '--max-stddev',
     type=float, default=4,
-    help='initial max allowable relative standard deviation')
+    help="initial max allowable relative standard deviation")
 __argparse.add_argument('-x', '--suffix',
-    help='suffix to append on config (e.g. "_before", "_after")')
+    help="suffix to append on config (e.g. '_before', '_after')")
 __argparse.add_argument('-w','--write-path',
-    help='directory to save .png proofs to disk.')
+    help="directory to save .png proofs to disk.")
 __argparse.add_argument('-v','--verbosity',
-    type=int, default=0, help='level of verbosity (0=none to 5=debug)')
+    type=int, default=1, help="level of verbosity (0=none to 5=debug)")
 __argparse.add_argument('-n', '--samples',
-    type=int, help='number of samples to collect for each bench')
+    type=int, help="number of samples to collect for each bench")
 __argparse.add_argument('-d', '--sample-ms',
-    type=int, help='duration of each sample')
+    type=int, help="duration of each sample")
 __argparse.add_argument('--fps',
-    action='store_true', help='use fps instead of ms')
+    action='store_true', help="use fps instead of ms")
 __argparse.add_argument('-c', '--config',
-    default='gpu', help='comma- or space-separated list of GPU configs')
+    default='gpu', help="comma- or space-separated list of GPU configs")
 __argparse.add_argument('skps',
     nargs='+',
-    help='.skp files or directories to expand for .skp files')
+    help=".skp files or directories to expand for .skp files")
 
 FLAGS = __argparse.parse_args()
 if FLAGS.adb:
   import _adb_path as _path
-  _path.set_device_serial(FLAGS.device_serial)
+  _path.init(FLAGS.device_serial)
 else:
   import _os_path as _path
 
 
 class StddevException(Exception):
   pass
 
 class Message:
   READLINE = 0,
-  EXIT = 1
+  POLL_HARDWARE = 1,
+  EXIT = 2
   def __init__(self, message, value=None):
     self.message = message
     self.value = value
 
-class SKPBench(Thread):
+class SubprocessMonitor(Thread):
+  def __init__(self, queue, proc):
+    self._queue = queue
+    self._proc = proc
+    Thread.__init__(self)
+
+  def run(self):
+    '''Runs on the background thread.'''
+    for line in iter(self._proc.stdout.readline, b''):
+      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
+    self._queue.put(Message(Message.EXIT))
+
+class SKPBench:
   ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)]
   if FLAGS.samples:
     ARGV.extend(['--samples', str(FLAGS.samples)])
   if FLAGS.sample_ms:
     ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
   if FLAGS.fps:
     ARGV.extend(['--fps', 'true'])
   if FLAGS.path:
     ARGV[0] = _path.join(FLAGS.path, ARGV[0])
   if FLAGS.adb:
     if FLAGS.device_serial is None:
       ARGV = ['adb', 'shell'] + ARGV
     else:
       ARGV = ['adb', '-s', FLAGS.device_serial, 'shell'] + ARGV
 
   @classmethod
   def print_header(cls):
     subprocess.call(cls.ARGV + ['--samples', '0'])
 
   def __init__(self, skp, config, max_stddev, best_result=None):
     self.skp = skp
     self.config = config
     self.max_stddev = max_stddev
     self.best_result = best_result
     self._queue = Queue()
-    Thread.__init__(self)
+    self._proc = None
+    self._monitor = None
+    self._hw_poll_timer = None
 
-  def execute(self):
-    self.start()
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    if self._proc:
+      self.terminate()
+    if self._hw_poll_timer:
+      self._hw_poll_timer.cancel()
+
+  def execute(self, hardware):
+    hardware.sanity_check()
+    self._schedule_hardware_poll()
+
+    commandline = self.ARGV + ['--config', self.config,
+                               '--skp', self.skp,
+                               '--suppressHeader', 'true']
+    if FLAGS.write_path:
+      pngfile = _path.join(FLAGS.write_path, self.config,
+                           _path.basename(self.skp) + '.png')
+      commandline.extend(['--png', pngfile])
+    if (FLAGS.verbosity >= 4):
+      quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
+      print(' '.join(quoted), file=sys.stderr)
+    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE)
+    self._monitor = SubprocessMonitor(self._queue, self._proc)
+    self._monitor.start()
+
     while True:
       message = self._queue.get()
       if message.message == Message.READLINE:
         result = BenchResult.match(message.value)
         if result:
-          self.__process_result(result)
+          hardware.sanity_check()
+          self._process_result(result)
         else:
           print(message.value)
         sys.stdout.flush()
         continue
+      if message.message == Message.POLL_HARDWARE:
+        hardware.sanity_check()
+        self._schedule_hardware_poll()
+        continue
       if message.message == Message.EXIT:
-        self.join()
+        self._monitor.join()
+        self._proc.wait()
+        if self._proc.returncode != 0:
+          raise Exception("skpbench exited with nonzero exit code %i" %
+                          self._proc.returncode)
+        self._proc = None
         break
 
-  def __process_result(self, result):
+  def _schedule_hardware_poll(self):
+    if self._hw_poll_timer:
+      self._hw_poll_timer.cancel()
+    self._hw_poll_timer = \
+      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
+    self._hw_poll_timer.start()
+
+  def _process_result(self, result):
     if not self.best_result or result.stddev <= self.best_result.stddev:
       self.best_result = result
-    elif FLAGS.verbosity >= 1:
-      print('NOTE: reusing previous result for %s/%s with lower stddev '
-            '(%s%% instead of %s%%).' %
+    elif FLAGS.verbosity >= 2:
+      print("reusing previous result for %s/%s with lower stddev "
+            "(%s%% instead of %s%%)." %
             (result.config, result.bench, self.best_result.stddev,
              result.stddev), file=sys.stderr)
     if self.max_stddev and self.best_result.stddev > self.max_stddev:
       raise StddevException()
-    self.best_result.print_values(config_suffix=FLAGS.suffix)
 
-  def run(self):
-    """Called on the background thread.
-
-    Launches and reads output from an skpbench process.
-
-    """
-    commandline = self.ARGV + ['--config', self.config,
-                               '--skp', self.skp,
-                               '--suppressHeader', 'true']
-    if (FLAGS.write_path):
-      pngfile = _path.join(FLAGS.write_path, self.config,
-                           _path.basename(self.skp) + '.png')
-      commandline.extend(['--png', pngfile])
-    if (FLAGS.verbosity >= 3):
-      print(' '.join(commandline), file=sys.stderr)
-    proc = subprocess.Popen(commandline, stdout=subprocess.PIPE)
-    for line in iter(proc.stdout.readline, b''):
-      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
-    proc.wait()
-    self._queue.put(Message(Message.EXIT, proc.returncode))
+  def terminate(self):
+    if self._proc:
+      self._proc.kill()
+      self._monitor.join()
+      self._proc.wait()
+      self._proc = None
 
 
-def main():
+def run_benchmarks(configs, skps, hardware):
   SKPBench.print_header()
 
-  # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)).
-  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
-  configs = re.split(DELIMITER, FLAGS.config)
-  skps = _path.find_skps(FLAGS.skps)
-
   benches = collections.deque([(skp, config, FLAGS.max_stddev)
                                for skp in skps
                                for config in configs])
   while benches:
     benchargs = benches.popleft()
-    skpbench = SKPBench(*benchargs)
-    try:
-      skpbench.execute()
+    with SKPBench(*benchargs) as skpbench:
+      try:
+        skpbench.execute(hardware)
+        if skpbench.best_result:
+          skpbench.best_result.print_values(config_suffix=FLAGS.suffix)
+        else:
+          print("WARNING: no result for %s with config %s" %
+                (skpbench.skp, skpbench.config), file=sys.stderr)
 
-    except StddevException:
-      retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
-      if FLAGS.verbosity >= 1:
-        print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). '
-              'Re-queuing with max=%.2f%%.' %
-              (skpbench.best_result.config, skpbench.best_result.bench,
-               skpbench.best_result.stddev, skpbench.max_stddev,
-               retry_max_stddev),
-              file=sys.stderr)
-      benches.append((skpbench.skp, skpbench.config, retry_max_stddev,
-                      skpbench.best_result))
+      except StddevException:
+        retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
+        if FLAGS.verbosity >= 2:
+          print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
+                "re-queuing with max=%.2f%%." %
+                (skpbench.best_result.config, skpbench.best_result.bench,
+                 skpbench.best_result.stddev, skpbench.max_stddev,
+                 retry_max_stddev),
+                file=sys.stderr)
+        benches.append((skpbench.skp, skpbench.config, retry_max_stddev,
+                        skpbench.best_result))
+
+      except HardwareException as exception:
+        skpbench.terminate()
+        naptime = max(hardware.kick_in_time, exception.sleeptime)
+        if FLAGS.verbosity >= 1:
+          print("%s; taking a %i second nap..." %
+                (exception.message, naptime), file=sys.stderr)
+        benches.appendleft(benchargs) # retry the same bench next time.
+        hardware.sleep(naptime - hardware.kick_in_time)
+        time.sleep(hardware.kick_in_time)
+
+
+def main():
+  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
+  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
+  configs = re.split(DELIMITER, FLAGS.config)
+  skps = _path.find_skps(FLAGS.skps)
+
+  if FLAGS.adb:
+    adb = Adb(FLAGS.device_serial)
+    model = adb.get_device_model()
+    if False:
+      pass # TODO: unique subclasses tailored to individual platforms.
+    else:
+      from _hardware_android import HardwareAndroid
+      print("WARNING: %s: don't know how to monitor this hardware; results "
+            "may be unreliable." % model, file=sys.stderr)
+      hardware = HardwareAndroid(adb)
+  else:
+    hardware = Hardware()
+
+  with hardware:
+    if hardware.kick_in_time:
+      print("sleeping %i seconds to allow hardware settings to kick in..." %
+            hardware.kick_in_time, file=sys.stderr)
+      time.sleep(hardware.kick_in_time)
+    run_benchmarks(configs, skps, hardware)
 
 
 if __name__ == '__main__':
   main()