| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 | 2 |
| 3 # Copyright 2016 Google Inc. | 3 # Copyright 2016 Google Inc. |
| 4 # | 4 # |
| 5 # Use of this source code is governed by a BSD-style license that can be | 5 # Use of this source code is governed by a BSD-style license that can be |
| 6 # found in the LICENSE file. | 6 # found in the LICENSE file. |
| 7 | 7 |
| 8 from __future__ import print_function | 8 from __future__ import print_function |
| 9 from _benchresult import BenchResult | 9 from _benchresult import BenchResult |
| 10 from argparse import ArgumentParser | 10 from argparse import ArgumentParser |
| 11 from os import path | |
| 12 from queue import Queue | 11 from queue import Queue |
| 13 from threading import Thread | 12 from threading import Thread |
| 14 import collections | 13 import collections |
| 15 import glob | 14 import glob |
| 16 import math | 15 import math |
| 17 import re | 16 import re |
| 18 import subprocess | 17 import subprocess |
| 19 import sys | 18 import sys |
| 20 | 19 |
| 21 __argparse = ArgumentParser(description=""" | 20 __argparse = ArgumentParser(description=""" |
| 22 | 21 |
| 23 Executes the skpbench binary with various configs and skps. | 22 Executes the skpbench binary with various configs and skps. |
| 24 | 23 |
| 25 Also monitors the output in order to filter out and re-run results that have an | 24 Also monitors the output in order to filter out and re-run results that have an |
| 26 unacceptable stddev. | 25 unacceptable stddev. |
| 27 | 26 |
| 28 """) | 27 """) |
| 29 | 28 |
| 29 __argparse.add_argument('--adb', |
| 30 action='store_true', help='execute skpbench over adb') |
| 31 __argparse.add_argument('-s', '--device-serial', |
| 32 help='if using adb, id of the specific device to target') |
| 30 __argparse.add_argument('-p', '--path', | 33 __argparse.add_argument('-p', '--path', |
| 31 help='directory to execute ./skpbench from') | 34 help='directory to execute ./skpbench from') |
| 32 __argparse.add_argument('-m', '--max-stddev', | 35 __argparse.add_argument('-m', '--max-stddev', |
| 33 type=float, default=4, | 36 type=float, default=4, |
| 34 help='initial max allowable relative standard deviation') | 37 help='initial max allowable relative standard deviation') |
| 35 __argparse.add_argument('-x', '--suffix', | 38 __argparse.add_argument('-x', '--suffix', |
| 36 help='suffix to append on config (e.g. "_before", "_after")') | 39 help='suffix to append on config (e.g. "_before", "_after")') |
| 37 __argparse.add_argument('-w','--write-path', | 40 __argparse.add_argument('-w','--write-path', |
| 38 help='directory to save .png proofs to disk.') | 41 help='directory to save .png proofs to disk.') |
| 39 __argparse.add_argument('-v','--verbosity', | 42 __argparse.add_argument('-v','--verbosity', |
| 40 type=int, default=0, help='level of verbosity (0=none to 5=debug)') | 43 type=int, default=0, help='level of verbosity (0=none to 5=debug)') |
| 41 __argparse.add_argument('-n', '--samples', | 44 __argparse.add_argument('-n', '--samples', |
| 42 type=int, help='number of samples to collect for each bench') | 45 type=int, help='number of samples to collect for each bench') |
| 43 __argparse.add_argument('-d', '--sample-ms', | 46 __argparse.add_argument('-d', '--sample-ms', |
| 44 type=int, help='duration of each sample') | 47 type=int, help='duration of each sample') |
| 45 __argparse.add_argument('--fps', | 48 __argparse.add_argument('--fps', |
| 46 action='store_true', help='use fps instead of ms') | 49 action='store_true', help='use fps instead of ms') |
| 47 __argparse.add_argument('-c', '--config', | 50 __argparse.add_argument('-c', '--config', |
| 48 default='gpu', help='comma- or space-separated list of GPU configs') | 51 default='gpu', help='comma- or space-separated list of GPU configs') |
| 49 __argparse.add_argument('skps', | 52 __argparse.add_argument('skps', |
| 50 nargs='+', | 53 nargs='+', |
| 51 help='.skp files or directories to expand for .skp files') | 54 help='.skp files or directories to expand for .skp files') |
| 52 | 55 |
| 53 FLAGS = __argparse.parse_args() | 56 FLAGS = __argparse.parse_args() |
| 57 if FLAGS.adb: |
| 58 import _adb_path as _path |
| 59 _path.set_device_serial(FLAGS.device_serial) |
| 60 else: |
| 61 import _os_path as _path |
| 54 | 62 |
| 55 | 63 |
| 56 class StddevException(Exception): | 64 class StddevException(Exception): |
| 57 pass | 65 pass |
| 58 | 66 |
| 59 class Message: | 67 class Message: |
| 60 READLINE = 0, | 68 READLINE = 0, |
| 61 EXIT = 1 | 69 EXIT = 1 |
| 62 def __init__(self, message, value=None): | 70 def __init__(self, message, value=None): |
| 63 self.message = message | 71 self.message = message |
| 64 self.value = value | 72 self.value = value |
| 65 | 73 |
| 66 class SKPBench(Thread): | 74 class SKPBench(Thread): |
| 67 ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] | 75 ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] |
| 68 if FLAGS.path: | |
| 69 ARGV[0] = path.join(FLAGS.path, ARGV[0]) | |
| 70 if FLAGS.samples: | 76 if FLAGS.samples: |
| 71 ARGV.extend(['--samples', str(FLAGS.samples)]) | 77 ARGV.extend(['--samples', str(FLAGS.samples)]) |
| 72 if FLAGS.sample_ms: | 78 if FLAGS.sample_ms: |
| 73 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) | 79 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) |
| 74 if FLAGS.fps: | 80 if FLAGS.fps: |
| 75 ARGV.extend(['--fps', 'true']) | 81 ARGV.extend(['--fps', 'true']) |
| 82 if FLAGS.path: |
| 83 ARGV[0] = _path.join(FLAGS.path, ARGV[0]) |
| 84 if FLAGS.adb: |
| 85 if FLAGS.device_serial is None: |
| 86 ARGV = ['adb', 'shell'] + ARGV |
| 87 else: |
| 88 ARGV = ['adb', '-s', FLAGS.device_serial, 'shell'] + ARGV |
| 76 | 89 |
| 77 @classmethod | 90 @classmethod |
| 78 def print_header(cls): | 91 def print_header(cls): |
| 79 subprocess.call(cls.ARGV + ['--samples', '0']) | 92 subprocess.call(cls.ARGV + ['--samples', '0']) |
| 80 | 93 |
| 81 def __init__(self, skp, config, max_stddev, best_result=None): | 94 def __init__(self, skp, config, max_stddev, best_result=None): |
| 82 self.skp = skp | 95 self.skp = skp |
| 83 self.config = config | 96 self.config = config |
| 84 self.max_stddev = max_stddev | 97 self.max_stddev = max_stddev |
| 85 self.best_result = best_result | 98 self.best_result = best_result |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 117 def run(self): | 130 def run(self): |
| 118 """Called on the background thread. | 131 """Called on the background thread. |
| 119 | 132 |
| 120 Launches and reads output from an skpbench process. | 133 Launches and reads output from an skpbench process. |
| 121 | 134 |
| 122 """ | 135 """ |
| 123 commandline = self.ARGV + ['--config', self.config, | 136 commandline = self.ARGV + ['--config', self.config, |
| 124 '--skp', self.skp, | 137 '--skp', self.skp, |
| 125 '--suppressHeader', 'true'] | 138 '--suppressHeader', 'true'] |
| 126 if (FLAGS.write_path): | 139 if (FLAGS.write_path): |
| 127 pngfile = path.join(FLAGS.write_path, self.config, | 140 pngfile = _path.join(FLAGS.write_path, self.config, |
| 128 path.basename(self.skp) + '.png') | 141 _path.basename(self.skp) + '.png') |
| 129 commandline.extend(['--png', pngfile]) | 142 commandline.extend(['--png', pngfile]) |
| 130 if (FLAGS.verbosity >= 3): | 143 if (FLAGS.verbosity >= 3): |
| 131 print(' '.join(commandline), file=sys.stderr) | 144 print(' '.join(commandline), file=sys.stderr) |
| 132 proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) | 145 proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) |
| 133 for line in iter(proc.stdout.readline, b''): | 146 for line in iter(proc.stdout.readline, b''): |
| 134 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) | 147 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) |
| 135 proc.wait() | 148 proc.wait() |
| 136 self._queue.put(Message(Message.EXIT, proc.returncode)) | 149 self._queue.put(Message(Message.EXIT, proc.returncode)) |
| 137 | 150 |
| 138 | 151 |
| 139 def main(): | 152 def main(): |
| 140 SKPBench.print_header() | 153 SKPBench.print_header() |
| 141 | 154 |
| 142 # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)). | 155 # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)). |
| 143 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' | 156 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' |
| 144 configs = re.split(DELIMITER, FLAGS.config) | 157 configs = re.split(DELIMITER, FLAGS.config) |
| 145 | 158 skps = _path.find_skps(FLAGS.skps) |
| 146 skps = list() | |
| 147 for skp in FLAGS.skps: | |
| 148 if (path.isdir(skp)): | |
| 149 skps.extend(glob.iglob(path.join(skp, '*.skp'))) | |
| 150 else: | |
| 151 skps.append(skp) | |
| 152 | 159 |
| 153 benches = collections.deque([(skp, config, FLAGS.max_stddev) | 160 benches = collections.deque([(skp, config, FLAGS.max_stddev) |
| 154 for skp in skps | 161 for skp in skps |
| 155 for config in configs]) | 162 for config in configs]) |
| 156 while benches: | 163 while benches: |
| 157 benchargs = benches.popleft() | 164 benchargs = benches.popleft() |
| 158 skpbench = SKPBench(*benchargs) | 165 skpbench = SKPBench(*benchargs) |
| 159 try: | 166 try: |
| 160 skpbench.execute() | 167 skpbench.execute() |
| 161 | 168 |
| 162 except StddevException: | 169 except StddevException: |
| 163 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) | 170 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) |
| 164 if FLAGS.verbosity >= 1: | 171 if FLAGS.verbosity >= 1: |
| 165 print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). ' | 172 print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). ' |
| 166 'Re-queuing with max=%.2f%%.' % | 173 'Re-queuing with max=%.2f%%.' % |
| 167 (skpbench.best_result.config, skpbench.best_result.bench, | 174 (skpbench.best_result.config, skpbench.best_result.bench, |
| 168 skpbench.best_result.stddev, skpbench.max_stddev, | 175 skpbench.best_result.stddev, skpbench.max_stddev, |
| 169 retry_max_stddev), | 176 retry_max_stddev), |
| 170 file=sys.stderr) | 177 file=sys.stderr) |
| 171 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, | 178 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, |
| 172 skpbench.best_result)) | 179 skpbench.best_result)) |
| 173 | 180 |
| 174 | 181 |
| 175 if __name__ == '__main__': | 182 if __name__ == '__main__': |
| 176 main() | 183 main() |
| OLD | NEW |