OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 # Copyright 2016 Google Inc. | 3 # Copyright 2016 Google Inc. |
4 # | 4 # |
5 # Use of this source code is governed by a BSD-style license that can be | 5 # Use of this source code is governed by a BSD-style license that can be |
6 # found in the LICENSE file. | 6 # found in the LICENSE file. |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
| 9 from _adb import Adb |
9 from _benchresult import BenchResult | 10 from _benchresult import BenchResult |
| 11 from _hardware import HardwareException, Hardware |
10 from argparse import ArgumentParser | 12 from argparse import ArgumentParser |
11 from queue import Queue | 13 from queue import Queue |
12 from threading import Thread | 14 from threading import Thread, Timer |
13 import collections | 15 import collections |
14 import glob | 16 import glob |
15 import math | 17 import math |
16 import re | 18 import re |
17 import subprocess | 19 import subprocess |
18 import sys | 20 import sys |
| 21 import time |
19 | 22 |
20 __argparse = ArgumentParser(description=""" | 23 __argparse = ArgumentParser(description=''' |
21 | 24 |
22 Executes the skpbench binary with various configs and skps. | 25 Executes the skpbench binary with various configs and skps. |
23 | 26 |
24 Also monitors the output in order to filter out and re-run results that have an | 27 Also monitors the output in order to filter out and re-run results that have an |
25 unacceptable stddev. | 28 unacceptable stddev. |
26 | 29 |
27 """) | 30 ''') |
28 | 31 |
29 __argparse.add_argument('--adb', | 32 __argparse.add_argument('--adb', |
30 action='store_true', help='execute skpbench over adb') | 33 action='store_true', help="execute skpbench over adb") |
31 __argparse.add_argument('-s', '--device-serial', | 34 __argparse.add_argument('-s', '--device-serial', |
32 help='if using adb, id of the specific device to target') | 35 help="if using adb, id of the specific device to target") |
33 __argparse.add_argument('-p', '--path', | 36 __argparse.add_argument('-p', '--path', |
34 help='directory to execute ./skpbench from') | 37 help="directory to execute ./skpbench from") |
35 __argparse.add_argument('-m', '--max-stddev', | 38 __argparse.add_argument('-m', '--max-stddev', |
36 type=float, default=4, | 39 type=float, default=4, |
37 help='initial max allowable relative standard deviation') | 40 help="initial max allowable relative standard deviation") |
38 __argparse.add_argument('-x', '--suffix', | 41 __argparse.add_argument('-x', '--suffix', |
39 help='suffix to append on config (e.g. "_before", "_after")') | 42 help="suffix to append on config (e.g. '_before', '_after')") |
40 __argparse.add_argument('-w','--write-path', | 43 __argparse.add_argument('-w','--write-path', |
41 help='directory to save .png proofs to disk.') | 44 help="directory to save .png proofs to disk.") |
42 __argparse.add_argument('-v','--verbosity', | 45 __argparse.add_argument('-v','--verbosity', |
43 type=int, default=0, help='level of verbosity (0=none to 5=debug)') | 46 type=int, default=1, help="level of verbosity (0=none to 5=debug)") |
44 __argparse.add_argument('-n', '--samples', | 47 __argparse.add_argument('-n', '--samples', |
45 type=int, help='number of samples to collect for each bench') | 48 type=int, help="number of samples to collect for each bench") |
46 __argparse.add_argument('-d', '--sample-ms', | 49 __argparse.add_argument('-d', '--sample-ms', |
47 type=int, help='duration of each sample') | 50 type=int, help="duration of each sample") |
48 __argparse.add_argument('--fps', | 51 __argparse.add_argument('--fps', |
49 action='store_true', help='use fps instead of ms') | 52 action='store_true', help="use fps instead of ms") |
50 __argparse.add_argument('-c', '--config', | 53 __argparse.add_argument('-c', '--config', |
51 default='gpu', help='comma- or space-separated list of GPU configs') | 54 default='gpu', help="comma- or space-separated list of GPU configs") |
52 __argparse.add_argument('skps', | 55 __argparse.add_argument('skps', |
53 nargs='+', | 56 nargs='+', |
54 help='.skp files or directories to expand for .skp files') | 57 help=".skp files or directories to expand for .skp files") |
55 | 58 |
56 FLAGS = __argparse.parse_args() | 59 FLAGS = __argparse.parse_args() |
57 if FLAGS.adb: | 60 if FLAGS.adb: |
58 import _adb_path as _path | 61 import _adb_path as _path |
59 _path.set_device_serial(FLAGS.device_serial) | 62 _path.init(FLAGS.device_serial) |
60 else: | 63 else: |
61 import _os_path as _path | 64 import _os_path as _path |
62 | 65 |
63 | 66 |
64 class StddevException(Exception): | 67 class StddevException(Exception): |
65 pass | 68 pass |
66 | 69 |
67 class Message: | 70 class Message: |
68 READLINE = 0, | 71 READLINE = 0, |
69 EXIT = 1 | 72 POLL_HARDWARE = 1, |
| 73 EXIT = 2 |
70 def __init__(self, message, value=None): | 74 def __init__(self, message, value=None): |
71 self.message = message | 75 self.message = message |
72 self.value = value | 76 self.value = value |
73 | 77 |
74 class SKPBench(Thread): | 78 class SubprocessMonitor(Thread): |
| 79 def __init__(self, queue, proc): |
| 80 self._queue = queue |
| 81 self._proc = proc |
| 82 Thread.__init__(self) |
| 83 |
| 84 def run(self): |
| 85 '''Runs on the background thread.''' |
| 86 for line in iter(self._proc.stdout.readline, b''): |
| 87 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) |
| 88 self._queue.put(Message(Message.EXIT)) |
| 89 |
| 90 class SKPBench: |
75 ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] | 91 ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] |
76 if FLAGS.samples: | 92 if FLAGS.samples: |
77 ARGV.extend(['--samples', str(FLAGS.samples)]) | 93 ARGV.extend(['--samples', str(FLAGS.samples)]) |
78 if FLAGS.sample_ms: | 94 if FLAGS.sample_ms: |
79 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) | 95 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) |
80 if FLAGS.fps: | 96 if FLAGS.fps: |
81 ARGV.extend(['--fps', 'true']) | 97 ARGV.extend(['--fps', 'true']) |
82 if FLAGS.path: | 98 if FLAGS.path: |
83 ARGV[0] = _path.join(FLAGS.path, ARGV[0]) | 99 ARGV[0] = _path.join(FLAGS.path, ARGV[0]) |
84 if FLAGS.adb: | 100 if FLAGS.adb: |
85 if FLAGS.device_serial is None: | 101 if FLAGS.device_serial is None: |
86 ARGV = ['adb', 'shell'] + ARGV | 102 ARGV = ['adb', 'shell'] + ARGV |
87 else: | 103 else: |
88 ARGV = ['adb', '-s', FLAGS.device_serial, 'shell'] + ARGV | 104 ARGV = ['adb', '-s', FLAGS.device_serial, 'shell'] + ARGV |
89 | 105 |
90 @classmethod | 106 @classmethod |
91 def print_header(cls): | 107 def print_header(cls): |
92 subprocess.call(cls.ARGV + ['--samples', '0']) | 108 subprocess.call(cls.ARGV + ['--samples', '0']) |
93 | 109 |
94 def __init__(self, skp, config, max_stddev, best_result=None): | 110 def __init__(self, skp, config, max_stddev, best_result=None): |
95 self.skp = skp | 111 self.skp = skp |
96 self.config = config | 112 self.config = config |
97 self.max_stddev = max_stddev | 113 self.max_stddev = max_stddev |
98 self.best_result = best_result | 114 self.best_result = best_result |
99 self._queue = Queue() | 115 self._queue = Queue() |
100 Thread.__init__(self) | 116 self._proc = None |
| 117 self._monitor = None |
| 118 self._hw_poll_timer = None |
101 | 119 |
102 def execute(self): | 120 def __enter__(self): |
103 self.start() | 121 return self |
| 122 |
| 123 def __exit__(self, exception_type, exception_value, traceback): |
| 124 if self._proc: |
| 125 self.terminate() |
| 126 if self._hw_poll_timer: |
| 127 self._hw_poll_timer.cancel() |
| 128 |
| 129 def execute(self, hardware): |
| 130 hardware.sanity_check() |
| 131 self._schedule_hardware_poll() |
| 132 |
| 133 commandline = self.ARGV + ['--config', self.config, |
| 134 '--skp', self.skp, |
| 135 '--suppressHeader', 'true'] |
| 136 if FLAGS.write_path: |
| 137 pngfile = _path.join(FLAGS.write_path, self.config, |
| 138 _path.basename(self.skp) + '.png') |
| 139 commandline.extend(['--png', pngfile]) |
| 140 if (FLAGS.verbosity >= 4): |
| 141 quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline] |
| 142 print(' '.join(quoted), file=sys.stderr) |
| 143 self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) |
| 144 self._monitor = SubprocessMonitor(self._queue, self._proc) |
| 145 self._monitor.start() |
| 146 |
104 while True: | 147 while True: |
105 message = self._queue.get() | 148 message = self._queue.get() |
106 if message.message == Message.READLINE: | 149 if message.message == Message.READLINE: |
107 result = BenchResult.match(message.value) | 150 result = BenchResult.match(message.value) |
108 if result: | 151 if result: |
109 self.__process_result(result) | 152 hardware.sanity_check() |
| 153 self._process_result(result) |
110 else: | 154 else: |
111 print(message.value) | 155 print(message.value) |
112 sys.stdout.flush() | 156 sys.stdout.flush() |
113 continue | 157 continue |
| 158 if message.message == Message.POLL_HARDWARE: |
| 159 hardware.sanity_check() |
| 160 self._schedule_hardware_poll() |
| 161 continue |
114 if message.message == Message.EXIT: | 162 if message.message == Message.EXIT: |
115 self.join() | 163 self._monitor.join() |
| 164 self._proc.wait() |
| 165 if self._proc.returncode != 0: |
| 166 raise Exception("skpbench exited with nonzero exit code %i" % |
| 167 self._proc.returncode) |
| 168 self._proc = None |
116 break | 169 break |
117 | 170 |
118 def __process_result(self, result): | 171 def _schedule_hardware_poll(self): |
| 172 if self._hw_poll_timer: |
| 173 self._hw_poll_timer.cancel() |
| 174 self._hw_poll_timer = \ |
| 175 Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE))) |
| 176 self._hw_poll_timer.start() |
| 177 |
| 178 def _process_result(self, result): |
119 if not self.best_result or result.stddev <= self.best_result.stddev: | 179 if not self.best_result or result.stddev <= self.best_result.stddev: |
120 self.best_result = result | 180 self.best_result = result |
121 elif FLAGS.verbosity >= 1: | 181 elif FLAGS.verbosity >= 2: |
122 print('NOTE: reusing previous result for %s/%s with lower stddev ' | 182 print("reusing previous result for %s/%s with lower stddev " |
123 '(%s%% instead of %s%%).' % | 183 "(%s%% instead of %s%%)." % |
124 (result.config, result.bench, self.best_result.stddev, | 184 (result.config, result.bench, self.best_result.stddev, |
125 result.stddev), file=sys.stderr) | 185 result.stddev), file=sys.stderr) |
126 if self.max_stddev and self.best_result.stddev > self.max_stddev: | 186 if self.max_stddev and self.best_result.stddev > self.max_stddev: |
127 raise StddevException() | 187 raise StddevException() |
128 self.best_result.print_values(config_suffix=FLAGS.suffix) | |
129 | 188 |
130 def run(self): | 189 def terminate(self): |
131 """Called on the background thread. | 190 if self._proc: |
132 | 191 self._proc.kill() |
133 Launches and reads output from an skpbench process. | 192 self._monitor.join() |
134 | 193 self._proc.wait() |
135 """ | 194 self._proc = None |
136 commandline = self.ARGV + ['--config', self.config, | |
137 '--skp', self.skp, | |
138 '--suppressHeader', 'true'] | |
139 if (FLAGS.write_path): | |
140 pngfile = _path.join(FLAGS.write_path, self.config, | |
141 _path.basename(self.skp) + '.png') | |
142 commandline.extend(['--png', pngfile]) | |
143 if (FLAGS.verbosity >= 3): | |
144 print(' '.join(commandline), file=sys.stderr) | |
145 proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) | |
146 for line in iter(proc.stdout.readline, b''): | |
147 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) | |
148 proc.wait() | |
149 self._queue.put(Message(Message.EXIT, proc.returncode)) | |
150 | 195 |
151 | 196 |
152 def main(): | 197 def run_benchmarks(configs, skps, hardware): |
153 SKPBench.print_header() | 198 SKPBench.print_header() |
154 | 199 |
155 # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)). | |
156 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' | |
157 configs = re.split(DELIMITER, FLAGS.config) | |
158 skps = _path.find_skps(FLAGS.skps) | |
159 | |
160 benches = collections.deque([(skp, config, FLAGS.max_stddev) | 200 benches = collections.deque([(skp, config, FLAGS.max_stddev) |
161 for skp in skps | 201 for skp in skps |
162 for config in configs]) | 202 for config in configs]) |
163 while benches: | 203 while benches: |
164 benchargs = benches.popleft() | 204 benchargs = benches.popleft() |
165 skpbench = SKPBench(*benchargs) | 205 with SKPBench(*benchargs) as skpbench: |
166 try: | 206 try: |
167 skpbench.execute() | 207 skpbench.execute(hardware) |
| 208 if skpbench.best_result: |
| 209 skpbench.best_result.print_values(config_suffix=FLAGS.suffix) |
| 210 else: |
| 211 print("WARNING: no result for %s with config %s" % |
| 212 (skpbench.skp, skpbench.config), file=sys.stderr) |
168 | 213 |
169 except StddevException: | 214 except StddevException: |
170 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) | 215 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) |
171 if FLAGS.verbosity >= 1: | 216 if FLAGS.verbosity >= 2: |
172 print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). ' | 217 print("stddev is too high for %s/%s (%s%%, max=%.2f%%), " |
173 'Re-queuing with max=%.2f%%.' % | 218 "re-queuing with max=%.2f%%." % |
174 (skpbench.best_result.config, skpbench.best_result.bench, | 219 (skpbench.best_result.config, skpbench.best_result.bench, |
175 skpbench.best_result.stddev, skpbench.max_stddev, | 220 skpbench.best_result.stddev, skpbench.max_stddev, |
176 retry_max_stddev), | 221 retry_max_stddev), |
177 file=sys.stderr) | 222 file=sys.stderr) |
178 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, | 223 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, |
179 skpbench.best_result)) | 224 skpbench.best_result)) |
| 225 |
| 226 except HardwareException as exception: |
| 227 skpbench.terminate() |
| 228 naptime = max(hardware.kick_in_time, exception.sleeptime) |
| 229 if FLAGS.verbosity >= 1: |
| 230 print("%s; taking a %i second nap..." % |
| 231 (exception.message, naptime), file=sys.stderr) |
| 232 benches.appendleft(benchargs) # retry the same bench next time. |
| 233 hardware.sleep(naptime - hardware.kick_in_time) |
| 234 time.sleep(hardware.kick_in_time) |
| 235 |
| 236 |
| 237 def main(): |
| 238 # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)). |
| 239 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' |
| 240 configs = re.split(DELIMITER, FLAGS.config) |
| 241 skps = _path.find_skps(FLAGS.skps) |
| 242 |
| 243 if FLAGS.adb: |
| 244 adb = Adb(FLAGS.device_serial) |
| 245 model = adb.get_device_model() |
| 246 if False: |
| 247 pass # TODO: unique subclasses tailored to individual platforms. |
| 248 else: |
| 249 from _hardware_android import HardwareAndroid |
| 250 print("WARNING: %s: don't know how to monitor this hardware; results " |
| 251 "may be unreliable." % model, file=sys.stderr) |
| 252 hardware = HardwareAndroid(adb) |
| 253 else: |
| 254 hardware = Hardware() |
| 255 |
| 256 with hardware: |
| 257 if hardware.kick_in_time: |
| 258 print("sleeping %i seconds to allow hardware settings to kick in..." % |
| 259 hardware.kick_in_time, file=sys.stderr) |
| 260 time.sleep(hardware.kick_in_time) |
| 261 run_benchmarks(configs, skps, hardware) |
180 | 262 |
181 | 263 |
182 if __name__ == '__main__': | 264 if __name__ == '__main__': |
183 main() | 265 main() |
OLD | NEW |