OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 # Copyright 2016 Google Inc. | 3 # Copyright 2016 Google Inc. |
4 # | 4 # |
5 # Use of this source code is governed by a BSD-style license that can be | 5 # Use of this source code is governed by a BSD-style license that can be |
6 # found in the LICENSE file. | 6 # found in the LICENSE file. |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 from _benchresult import BenchResult | 9 from _benchresult import BenchResult |
10 from argparse import ArgumentParser | 10 from argparse import ArgumentParser |
11 from os import path | |
12 from queue import Queue | 11 from queue import Queue |
13 from threading import Thread | 12 from threading import Thread |
14 import collections | 13 import collections |
15 import glob | 14 import glob |
16 import math | 15 import math |
17 import re | 16 import re |
18 import subprocess | 17 import subprocess |
19 import sys | 18 import sys |
20 | 19 |
21 __argparse = ArgumentParser(description=""" | 20 __argparse = ArgumentParser(description=""" |
22 | 21 |
23 Executes the skpbench binary with various configs and skps. | 22 Executes the skpbench binary with various configs and skps. |
24 | 23 |
25 Also monitors the output in order to filter out and re-run results that have an | 24 Also monitors the output in order to filter out and re-run results that have an |
26 unacceptable stddev. | 25 unacceptable stddev. |
27 | 26 |
28 """) | 27 """) |
29 | 28 |
| 29 __argparse.add_argument('--adb', |
| 30 action='store_true', help='execute skpbench over adb') |
| 31 __argparse.add_argument('-s', '--device-serial', |
| 32 help='if using adb, id of the specific device to target') |
30 __argparse.add_argument('-p', '--path', | 33 __argparse.add_argument('-p', '--path', |
31 help='directory to execute ./skpbench from') | 34 help='directory to execute ./skpbench from') |
32 __argparse.add_argument('-m', '--max-stddev', | 35 __argparse.add_argument('-m', '--max-stddev', |
33 type=float, default=4, | 36 type=float, default=4, |
34 help='initial max allowable relative standard deviation') | 37 help='initial max allowable relative standard deviation') |
35 __argparse.add_argument('-x', '--suffix', | 38 __argparse.add_argument('-x', '--suffix', |
36 help='suffix to append on config (e.g. "_before", "_after")') | 39 help='suffix to append on config (e.g. "_before", "_after")') |
37 __argparse.add_argument('-w','--write-path', | 40 __argparse.add_argument('-w','--write-path', |
38 help='directory to save .png proofs to disk.') | 41 help='directory to save .png proofs to disk.') |
39 __argparse.add_argument('-v','--verbosity', | 42 __argparse.add_argument('-v','--verbosity', |
40 type=int, default=0, help='level of verbosity (0=none to 5=debug)') | 43 type=int, default=0, help='level of verbosity (0=none to 5=debug)') |
41 __argparse.add_argument('-n', '--samples', | 44 __argparse.add_argument('-n', '--samples', |
42 type=int, help='number of samples to collect for each bench') | 45 type=int, help='number of samples to collect for each bench') |
43 __argparse.add_argument('-d', '--sample-ms', | 46 __argparse.add_argument('-d', '--sample-ms', |
44 type=int, help='duration of each sample') | 47 type=int, help='duration of each sample') |
45 __argparse.add_argument('--fps', | 48 __argparse.add_argument('--fps', |
46 action='store_true', help='use fps instead of ms') | 49 action='store_true', help='use fps instead of ms') |
47 __argparse.add_argument('-c', '--config', | 50 __argparse.add_argument('-c', '--config', |
48 default='gpu', help='comma- or space-separated list of GPU configs') | 51 default='gpu', help='comma- or space-separated list of GPU configs') |
49 __argparse.add_argument('skps', | 52 __argparse.add_argument('skps', |
50 nargs='+', | 53 nargs='+', |
51 help='.skp files or directories to expand for .skp files') | 54 help='.skp files or directories to expand for .skp files') |
52 | 55 |
53 FLAGS = __argparse.parse_args() | 56 FLAGS = __argparse.parse_args() |
| 57 if FLAGS.adb: |
| 58 import _adb_path as _path |
| 59 _path.set_device_serial(FLAGS.device_serial) |
| 60 else: |
| 61 import _os_path as _path |
54 | 62 |
55 | 63 |
56 class StddevException(Exception): | 64 class StddevException(Exception): |
57 pass | 65 pass |
58 | 66 |
59 class Message: | 67 class Message: |
60 READLINE = 0, | 68 READLINE = 0, |
61 EXIT = 1 | 69 EXIT = 1 |
62 def __init__(self, message, value=None): | 70 def __init__(self, message, value=None): |
63 self.message = message | 71 self.message = message |
64 self.value = value | 72 self.value = value |
65 | 73 |
66 class SKPBench(Thread): | 74 class SKPBench(Thread): |
67 ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] | 75 ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] |
68 if FLAGS.path: | |
69 ARGV[0] = path.join(FLAGS.path, ARGV[0]) | |
70 if FLAGS.samples: | 76 if FLAGS.samples: |
71 ARGV.extend(['--samples', str(FLAGS.samples)]) | 77 ARGV.extend(['--samples', str(FLAGS.samples)]) |
72 if FLAGS.sample_ms: | 78 if FLAGS.sample_ms: |
73 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) | 79 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) |
74 if FLAGS.fps: | 80 if FLAGS.fps: |
75 ARGV.extend(['--fps', 'true']) | 81 ARGV.extend(['--fps', 'true']) |
| 82 if FLAGS.path: |
| 83 ARGV[0] = _path.join(FLAGS.path, ARGV[0]) |
| 84 if FLAGS.adb: |
| 85 if FLAGS.device_serial is None: |
| 86 ARGV = ['adb', 'shell'] + ARGV |
| 87 else: |
| 88 ARGV = ['adb', '-s', FLAGS.device_serial, 'shell'] + ARGV |
76 | 89 |
77 @classmethod | 90 @classmethod |
78 def print_header(cls): | 91 def print_header(cls): |
79 subprocess.call(cls.ARGV + ['--samples', '0']) | 92 subprocess.call(cls.ARGV + ['--samples', '0']) |
80 | 93 |
81 def __init__(self, skp, config, max_stddev, best_result=None): | 94 def __init__(self, skp, config, max_stddev, best_result=None): |
82 self.skp = skp | 95 self.skp = skp |
83 self.config = config | 96 self.config = config |
84 self.max_stddev = max_stddev | 97 self.max_stddev = max_stddev |
85 self.best_result = best_result | 98 self.best_result = best_result |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
117 def run(self): | 130 def run(self): |
118 """Called on the background thread. | 131 """Called on the background thread. |
119 | 132 |
120 Launches and reads output from an skpbench process. | 133 Launches and reads output from an skpbench process. |
121 | 134 |
122 """ | 135 """ |
123 commandline = self.ARGV + ['--config', self.config, | 136 commandline = self.ARGV + ['--config', self.config, |
124 '--skp', self.skp, | 137 '--skp', self.skp, |
125 '--suppressHeader', 'true'] | 138 '--suppressHeader', 'true'] |
126 if (FLAGS.write_path): | 139 if (FLAGS.write_path): |
127 pngfile = path.join(FLAGS.write_path, self.config, | 140 pngfile = _path.join(FLAGS.write_path, self.config, |
128 path.basename(self.skp) + '.png') | 141 _path.basename(self.skp) + '.png') |
129 commandline.extend(['--png', pngfile]) | 142 commandline.extend(['--png', pngfile]) |
130 if (FLAGS.verbosity >= 3): | 143 if (FLAGS.verbosity >= 3): |
131 print(' '.join(commandline), file=sys.stderr) | 144 print(' '.join(commandline), file=sys.stderr) |
132 proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) | 145 proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) |
133 for line in iter(proc.stdout.readline, b''): | 146 for line in iter(proc.stdout.readline, b''): |
134 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) | 147 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) |
135 proc.wait() | 148 proc.wait() |
136 self._queue.put(Message(Message.EXIT, proc.returncode)) | 149 self._queue.put(Message(Message.EXIT, proc.returncode)) |
137 | 150 |
138 | 151 |
139 def main(): | 152 def main(): |
140 SKPBench.print_header() | 153 SKPBench.print_header() |
141 | 154 |
142 # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)). | 155 # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)). |
143 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' | 156 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' |
144 configs = re.split(DELIMITER, FLAGS.config) | 157 configs = re.split(DELIMITER, FLAGS.config) |
145 | 158 skps = _path.find_skps(FLAGS.skps) |
146 skps = list() | |
147 for skp in FLAGS.skps: | |
148 if (path.isdir(skp)): | |
149 skps.extend(glob.iglob(path.join(skp, '*.skp'))) | |
150 else: | |
151 skps.append(skp) | |
152 | 159 |
153 benches = collections.deque([(skp, config, FLAGS.max_stddev) | 160 benches = collections.deque([(skp, config, FLAGS.max_stddev) |
154 for skp in skps | 161 for skp in skps |
155 for config in configs]) | 162 for config in configs]) |
156 while benches: | 163 while benches: |
157 benchargs = benches.popleft() | 164 benchargs = benches.popleft() |
158 skpbench = SKPBench(*benchargs) | 165 skpbench = SKPBench(*benchargs) |
159 try: | 166 try: |
160 skpbench.execute() | 167 skpbench.execute() |
161 | 168 |
162 except StddevException: | 169 except StddevException: |
163 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) | 170 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) |
164 if FLAGS.verbosity >= 1: | 171 if FLAGS.verbosity >= 1: |
165 print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). ' | 172 print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). ' |
166 'Re-queuing with max=%.2f%%.' % | 173 'Re-queuing with max=%.2f%%.' % |
167 (skpbench.best_result.config, skpbench.best_result.bench, | 174 (skpbench.best_result.config, skpbench.best_result.bench, |
168 skpbench.best_result.stddev, skpbench.max_stddev, | 175 skpbench.best_result.stddev, skpbench.max_stddev, |
169 retry_max_stddev), | 176 retry_max_stddev), |
170 file=sys.stderr) | 177 file=sys.stderr) |
171 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, | 178 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, |
172 skpbench.best_result)) | 179 skpbench.best_result)) |
173 | 180 |
174 | 181 |
175 if __name__ == '__main__': | 182 if __name__ == '__main__': |
176 main() | 183 main() |
OLD | NEW |