OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 # Copyright 2016 Google Inc. | 3 # Copyright 2016 Google Inc. |
4 # | 4 # |
5 # Use of this source code is governed by a BSD-style license that can be | 5 # Use of this source code is governed by a BSD-style license that can be |
6 # found in the LICENSE file. | 6 # found in the LICENSE file. |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 from _benchresult import BenchResult | 9 from _benchresult import BenchResult |
10 from argparse import ArgumentParser | 10 from argparse import ArgumentParser |
11 from collections import defaultdict, namedtuple | |
12 from datetime import datetime | 11 from datetime import datetime |
| 12 import collections |
13 import operator | 13 import operator |
14 import os | 14 import os |
15 import sys | 15 import sys |
16 import tempfile | 16 import tempfile |
17 import urllib | 17 import urllib |
18 import urlparse | 18 import urlparse |
19 import webbrowser | 19 import webbrowser |
20 | 20 |
21 __argparse = ArgumentParser(description=""" | 21 __argparse = ArgumentParser(description=""" |
22 | 22 |
23 Parses output files from skpbench.py into csv. | 23 Parses output files from skpbench.py into csv. |
24 | 24 |
25 This script can also be used to generate a Google sheet: | 25 This script can also be used to generate a Google sheet: |
26 | 26 |
27 (1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: | 27 (1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: |
28 https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegba
iigmenfmjfclcdgdpimamgkj | 28 https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegba
iigmenfmjfclcdgdpimamgkj |
29 | 29 |
30 (2) Update your global OS file associations to use Chrome for .csv files. | 30 (2) Designate Chrome os-wide as the default application for opening .csv files. |
31 | 31 |
32 (3) Run parseskpbench.py with the --open flag. | 32 (3) Run parseskpbench.py with the --open flag. |
33 | 33 |
34 """) | 34 """) |
35 | 35 |
36 __argparse.add_argument('-r', '--result', | 36 __argparse.add_argument('-r', '--result', |
37 choices=['accum', 'median', 'max', 'min'], default='accum', | 37 choices=['accum', 'median', 'max', 'min'], default='accum', |
38 help="result to use for cell values") | 38 help="result to use for cell values") |
39 __argparse.add_argument('-f', '--force', | 39 __argparse.add_argument('-f', '--force', |
40 action='store_true', help='silently ignore warnings') | 40 action='store_true', help='silently ignore warnings') |
41 __argparse.add_argument('-o', '--open', | 41 __argparse.add_argument('-o', '--open', |
42 action='store_true', | 42 action='store_true', |
43 help="generate a temp file and open it (theoretically in a web browser)") | 43 help="generate a temp file and open it (theoretically in a web browser)") |
44 __argparse.add_argument('-n', '--name', | 44 __argparse.add_argument('-n', '--name', |
45 default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), | 45 default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), |
46 help="if using --open, a name for the temp file") | 46 help="if using --open, a name for the temp file") |
47 __argparse.add_argument('sources', | 47 __argparse.add_argument('sources', |
48 nargs='+', help="source files with skpbench results ('-' for stdin)") | 48 nargs='+', help="source files with skpbench results ('-' for stdin)") |
49 | 49 |
50 FLAGS = __argparse.parse_args() | 50 FLAGS = __argparse.parse_args() |
51 | 51 |
52 RESULT_QUALIFIERS = ('sample_ms', 'clock', 'metric') | |
53 | |
54 class FullConfig(namedtuple('fullconfig', ('config',) + RESULT_QUALIFIERS)): | |
55 def qualified_name(self, qualifiers=RESULT_QUALIFIERS): | |
56 return get_qualified_name(self.config.replace(',', ' '), | |
57 {x:getattr(self, x) for x in qualifiers}) | |
58 | |
59 def get_qualified_name(name, qualifiers): | |
60 if not qualifiers: | |
61 return name | |
62 else: | |
63 args = ('%s=%s' % (k,v) for k,v in qualifiers.iteritems()) | |
64 return '%s (%s)' % (name, ' '.join(args)) | |
65 | 52 |
66 class Parser: | 53 class Parser: |
67 def __init__(self): | 54 def __init__(self): |
68 self.sheet_qualifiers = {x:None for x in RESULT_QUALIFIERS} | 55 self.configs = list() # use list to preserve the order configs appear in. |
69 self.config_qualifiers = set() | 56 self.rows = collections.defaultdict(dict) |
70 self.fullconfigs = list() # use list to preserve the order. | 57 self.cols = collections.defaultdict(dict) |
71 self.rows = defaultdict(dict) | 58 self.metric = None |
72 self.cols = defaultdict(dict) | 59 self.sample_ms = None |
73 | 60 |
74 def parse_file(self, infile): | 61 def parse_file(self, infile): |
75 for line in infile: | 62 for line in infile: |
76 match = BenchResult.match(line) | 63 match = BenchResult.match(line) |
77 if not match: | 64 if not match: |
78 continue | 65 continue |
79 | 66 if self.metric is None: |
80 fullconfig = FullConfig(*(match.get_string(x) | 67 self.metric = match.metric |
81 for x in FullConfig._fields)) | 68 elif match.metric != self.metric: |
82 if not fullconfig in self.fullconfigs: | 69 raise ValueError("results have mismatched metrics (%s and %s)" % |
83 self.fullconfigs.append(fullconfig) | 70 (self.metric, match.metric)) |
84 | 71 if self.sample_ms is None: |
85 for qualifier, value in self.sheet_qualifiers.items(): | 72 self.sample_ms = match.sample_ms |
86 if value is None: | 73 elif not FLAGS.force and match.sample_ms != self.sample_ms: |
87 self.sheet_qualifiers[qualifier] = match.get_string(qualifier) | 74 raise ValueError("results have mismatched sampling times. " |
88 elif value != match.get_string(qualifier): | 75 "(use --force to ignore)") |
89 del self.sheet_qualifiers[qualifier] | 76 if not match.config in self.configs: |
90 self.config_qualifiers.add(qualifier) | 77 self.configs.append(match.config) |
91 | 78 self.rows[match.bench][match.config] = match.get_string(FLAGS.result) |
92 self.rows[match.bench][fullconfig] = match.get_string(FLAGS.result) | 79 self.cols[match.config][match.bench] = getattr(match, FLAGS.result) |
93 self.cols[fullconfig][match.bench] = getattr(match, FLAGS.result) | |
94 | 80 |
95 def print_csv(self, outfile=sys.stdout): | 81 def print_csv(self, outfile=sys.stdout): |
96 # Write the title. | 82 print('%s_%s' % (FLAGS.result, self.metric), file=outfile) |
97 print(get_qualified_name(FLAGS.result, self.sheet_qualifiers), file=outfile) | |
98 | 83 |
99 # Write the header. | 84 # Write the header. |
100 outfile.write('bench,') | 85 outfile.write('bench,') |
101 for fullconfig in self.fullconfigs: | 86 for config in self.configs: |
102 outfile.write('%s,' % fullconfig.qualified_name(self.config_qualifiers)) | 87 outfile.write('%s,' % config) |
103 outfile.write('\n') | 88 outfile.write('\n') |
104 | 89 |
105 # Write the rows. | 90 # Write the rows. |
106 for bench, row in self.rows.iteritems(): | 91 for bench, row in self.rows.items(): |
107 outfile.write('%s,' % bench) | 92 outfile.write('%s,' % bench) |
108 for fullconfig in self.fullconfigs: | 93 for config in self.configs: |
109 if fullconfig in row: | 94 if config in row: |
110 outfile.write('%s,' % row[fullconfig]) | 95 outfile.write('%s,' % row[config]) |
111 elif FLAGS.force: | 96 elif FLAGS.force: |
112 outfile.write('NULL,') | 97 outfile.write(',') |
113 else: | 98 else: |
114 raise ValueError("%s: missing value for %s. (use --force to ignore)" % | 99 raise ValueError("%s: missing value for %s. (use --force to ignore)" % |
115 (bench, | 100 (bench, config)) |
116 fullconfig.qualified_name(self.config_qualifiers))) | |
117 outfile.write('\n') | 101 outfile.write('\n') |
118 | 102 |
119 # Add simple, literal averages. | 103 # Add simple, literal averages. |
120 if len(self.rows) > 1: | 104 if len(self.rows) > 1: |
121 outfile.write('\n') | 105 outfile.write('\n') |
122 self._print_computed_row('MEAN', | 106 self.__print_computed_row('MEAN', |
123 lambda col: reduce(operator.add, col.values()) / len(col), | 107 lambda col: reduce(operator.add, col.values()) / len(col), |
124 outfile=outfile) | 108 outfile=outfile) |
125 self._print_computed_row('GEOMEAN', | 109 self.__print_computed_row('GEOMEAN', |
126 lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), | 110 lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), |
127 outfile=outfile) | 111 outfile=outfile) |
128 | 112 |
129 def _print_computed_row(self, name, func, outfile=sys.stdout): | 113 def __print_computed_row(self, name, func, outfile=sys.stdout): |
130 outfile.write('%s,' % name) | 114 outfile.write('%s,' % name) |
131 for fullconfig in self.fullconfigs: | 115 for config in self.configs: |
132 if len(self.cols[fullconfig]) != len(self.rows): | 116 assert(len(self.cols[config]) == len(self.rows)) |
133 outfile.write('NULL,') | 117 outfile.write('%.4g,' % func(self.cols[config])) |
134 continue | |
135 outfile.write('%.4g,' % func(self.cols[fullconfig])) | |
136 outfile.write('\n') | 118 outfile.write('\n') |
137 | 119 |
| 120 |
138 def main(): | 121 def main(): |
139 parser = Parser() | 122 parser = Parser() |
140 | 123 |
141 # Parse the input files. | 124 # Parse the input files. |
142 for src in FLAGS.sources: | 125 for src in FLAGS.sources: |
143 if src == '-': | 126 if src == '-': |
144 parser.parse_file(sys.stdin) | 127 parser.parse_file(sys.stdin) |
145 else: | 128 else: |
146 with open(src, mode='r') as infile: | 129 with open(src, mode='r') as infile: |
147 parser.parse_file(infile) | 130 parser.parse_file(infile) |
148 | 131 |
149 # Print the csv. | 132 # Print the csv. |
150 if not FLAGS.open: | 133 if not FLAGS.open: |
151 parser.print_csv() | 134 parser.print_csv() |
152 else: | 135 else: |
153 dirname = tempfile.mkdtemp() | 136 dirname = tempfile.mkdtemp() |
154 basename = FLAGS.name | 137 basename = FLAGS.name |
155 if os.path.splitext(basename)[1] != '.csv': | 138 if os.path.splitext(basename)[1] != '.csv': |
156 basename += '.csv'; | 139 basename += '.csv'; |
157 pathname = os.path.join(dirname, basename) | 140 pathname = os.path.join(dirname, basename) |
158 with open(pathname, mode='w') as tmpfile: | 141 with open(pathname, mode='w') as tmpfile: |
159 parser.print_csv(outfile=tmpfile) | 142 parser.print_csv(outfile=tmpfile) |
160 fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) | 143 fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) |
161 print('opening %s' % fileuri) | 144 print('opening %s' % fileuri) |
162 webbrowser.open(fileuri) | 145 webbrowser.open(fileuri) |
163 | 146 |
164 | 147 |
165 if __name__ == '__main__': | 148 if __name__ == '__main__': |
166 main() | 149 main() |
OLD | NEW |