OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 | 2 |
3 # Copyright 2016 Google Inc. | 3 # Copyright 2016 Google Inc. |
4 # | 4 # |
5 # Use of this source code is governed by a BSD-style license that can be | 5 # Use of this source code is governed by a BSD-style license that can be |
6 # found in the LICENSE file. | 6 # found in the LICENSE file. |
7 | 7 |
8 from __future__ import print_function | 8 from __future__ import print_function |
9 from _benchresult import BenchResult | 9 from _benchresult import BenchResult |
10 from argparse import ArgumentParser | 10 from argparse import ArgumentParser |
| 11 from collections import defaultdict, namedtuple |
11 from datetime import datetime | 12 from datetime import datetime |
12 import collections | |
13 import operator | 13 import operator |
14 import os | 14 import os |
15 import sys | 15 import sys |
16 import tempfile | 16 import tempfile |
17 import urllib | 17 import urllib |
18 import urlparse | 18 import urlparse |
19 import webbrowser | 19 import webbrowser |
20 | 20 |
21 __argparse = ArgumentParser(description=""" | 21 __argparse = ArgumentParser(description=""" |
22 | 22 |
23 Parses output files from skpbench.py into csv. | 23 Parses output files from skpbench.py into csv. |
24 | 24 |
25 This script can also be used to generate a Google sheet: | 25 This script can also be used to generate a Google sheet: |
26 | 26 |
27 (1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: | 27 (1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: |
28 https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegba
iigmenfmjfclcdgdpimamgkj | 28 https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegba
iigmenfmjfclcdgdpimamgkj |
29 | 29 |
30 (2) Designate Chrome os-wide as the default application for opening .csv files. | 30 (2) Update your global OS file associations to use Chrome for .csv files. |
31 | 31 |
32 (3) Run parseskpbench.py with the --open flag. | 32 (3) Run parseskpbench.py with the --open flag. |
33 | 33 |
34 """) | 34 """) |
35 | 35 |
36 __argparse.add_argument('-r', '--result', | 36 __argparse.add_argument('-r', '--result', |
37 choices=['accum', 'median', 'max', 'min'], default='accum', | 37 choices=['accum', 'median', 'max', 'min'], default='accum', |
38 help="result to use for cell values") | 38 help="result to use for cell values") |
39 __argparse.add_argument('-f', '--force', | 39 __argparse.add_argument('-f', '--force', |
40 action='store_true', help='silently ignore warnings') | 40 action='store_true', help='silently ignore warnings') |
41 __argparse.add_argument('-o', '--open', | 41 __argparse.add_argument('-o', '--open', |
42 action='store_true', | 42 action='store_true', |
43 help="generate a temp file and open it (theoretically in a web browser)") | 43 help="generate a temp file and open it (theoretically in a web browser)") |
44 __argparse.add_argument('-n', '--name', | 44 __argparse.add_argument('-n', '--name', |
45 default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), | 45 default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), |
46 help="if using --open, a name for the temp file") | 46 help="if using --open, a name for the temp file") |
47 __argparse.add_argument('sources', | 47 __argparse.add_argument('sources', |
48 nargs='+', help="source files with skpbench results ('-' for stdin)") | 48 nargs='+', help="source files with skpbench results ('-' for stdin)") |
49 | 49 |
50 FLAGS = __argparse.parse_args() | 50 FLAGS = __argparse.parse_args() |
51 | 51 |
| 52 RESULT_QUALIFIERS = ('sample_ms', 'clock', 'metric') |
| 53 |
| 54 class FullConfig(namedtuple('fullconfig', ('config',) + RESULT_QUALIFIERS)): |
| 55 def qualified_name(self, qualifiers=RESULT_QUALIFIERS): |
| 56 return get_qualified_name(self.config.replace(',', ' '), |
| 57 {x:getattr(self, x) for x in qualifiers}) |
| 58 |
| 59 def get_qualified_name(name, qualifiers): |
| 60 if not qualifiers: |
| 61 return name |
| 62 else: |
| 63 args = ('%s=%s' % (k,v) for k,v in qualifiers.iteritems()) |
| 64 return '%s (%s)' % (name, ' '.join(args)) |
52 | 65 |
53 class Parser: | 66 class Parser: |
54 def __init__(self): | 67 def __init__(self): |
55 self.configs = list() # use list to preserve the order configs appear in. | 68 self.sheet_qualifiers = {x:None for x in RESULT_QUALIFIERS} |
56 self.rows = collections.defaultdict(dict) | 69 self.config_qualifiers = set() |
57 self.cols = collections.defaultdict(dict) | 70 self.fullconfigs = list() # use list to preserve the order. |
58 self.metric = None | 71 self.rows = defaultdict(dict) |
59 self.sample_ms = None | 72 self.cols = defaultdict(dict) |
60 | 73 |
61 def parse_file(self, infile): | 74 def parse_file(self, infile): |
62 for line in infile: | 75 for line in infile: |
63 match = BenchResult.match(line) | 76 match = BenchResult.match(line) |
64 if not match: | 77 if not match: |
65 continue | 78 continue |
66 if self.metric is None: | 79 |
67 self.metric = match.metric | 80 fullconfig = FullConfig(*(match.get_string(x) |
68 elif match.metric != self.metric: | 81 for x in FullConfig._fields)) |
69 raise ValueError("results have mismatched metrics (%s and %s)" % | 82 if not fullconfig in self.fullconfigs: |
70 (self.metric, match.metric)) | 83 self.fullconfigs.append(fullconfig) |
71 if self.sample_ms is None: | 84 |
72 self.sample_ms = match.sample_ms | 85 for qualifier, value in self.sheet_qualifiers.items(): |
73 elif not FLAGS.force and match.sample_ms != self.sample_ms: | 86 if value is None: |
74 raise ValueError("results have mismatched sampling times. " | 87 self.sheet_qualifiers[qualifier] = match.get_string(qualifier) |
75 "(use --force to ignore)") | 88 elif value != match.get_string(qualifier): |
76 if not match.config in self.configs: | 89 del self.sheet_qualifiers[qualifier] |
77 self.configs.append(match.config) | 90 self.config_qualifiers.add(qualifier) |
78 self.rows[match.bench][match.config] = match.get_string(FLAGS.result) | 91 |
79 self.cols[match.config][match.bench] = getattr(match, FLAGS.result) | 92 self.rows[match.bench][fullconfig] = match.get_string(FLAGS.result) |
| 93 self.cols[fullconfig][match.bench] = getattr(match, FLAGS.result) |
80 | 94 |
81 def print_csv(self, outfile=sys.stdout): | 95 def print_csv(self, outfile=sys.stdout): |
82 print('%s_%s' % (FLAGS.result, self.metric), file=outfile) | 96 # Write the title. |
| 97 print(get_qualified_name(FLAGS.result, self.sheet_qualifiers), file=outfile) |
83 | 98 |
84 # Write the header. | 99 # Write the header. |
85 outfile.write('bench,') | 100 outfile.write('bench,') |
86 for config in self.configs: | 101 for fullconfig in self.fullconfigs: |
87 outfile.write('%s,' % config) | 102 outfile.write('%s,' % fullconfig.qualified_name(self.config_qualifiers)) |
88 outfile.write('\n') | 103 outfile.write('\n') |
89 | 104 |
90 # Write the rows. | 105 # Write the rows. |
91 for bench, row in self.rows.items(): | 106 for bench, row in self.rows.iteritems(): |
92 outfile.write('%s,' % bench) | 107 outfile.write('%s,' % bench) |
93 for config in self.configs: | 108 for fullconfig in self.fullconfigs: |
94 if config in row: | 109 if fullconfig in row: |
95 outfile.write('%s,' % row[config]) | 110 outfile.write('%s,' % row[fullconfig]) |
96 elif FLAGS.force: | 111 elif FLAGS.force: |
97 outfile.write(',') | 112 outfile.write('NULL,') |
98 else: | 113 else: |
99 raise ValueError("%s: missing value for %s. (use --force to ignore)" % | 114 raise ValueError("%s: missing value for %s. (use --force to ignore)" % |
100 (bench, config)) | 115 (bench, |
| 116 fullconfig.qualified_name(self.config_qualifiers))) |
101 outfile.write('\n') | 117 outfile.write('\n') |
102 | 118 |
103 # Add simple, literal averages. | 119 # Add simple, literal averages. |
104 if len(self.rows) > 1: | 120 if len(self.rows) > 1: |
105 outfile.write('\n') | 121 outfile.write('\n') |
106 self.__print_computed_row('MEAN', | 122 self._print_computed_row('MEAN', |
107 lambda col: reduce(operator.add, col.values()) / len(col), | 123 lambda col: reduce(operator.add, col.values()) / len(col), |
108 outfile=outfile) | 124 outfile=outfile) |
109 self.__print_computed_row('GEOMEAN', | 125 self._print_computed_row('GEOMEAN', |
110 lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), | 126 lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), |
111 outfile=outfile) | 127 outfile=outfile) |
112 | 128 |
113 def __print_computed_row(self, name, func, outfile=sys.stdout): | 129 def _print_computed_row(self, name, func, outfile=sys.stdout): |
114 outfile.write('%s,' % name) | 130 outfile.write('%s,' % name) |
115 for config in self.configs: | 131 for fullconfig in self.fullconfigs: |
116 assert(len(self.cols[config]) == len(self.rows)) | 132 if len(self.cols[fullconfig]) != len(self.rows): |
117 outfile.write('%.4g,' % func(self.cols[config])) | 133 outfile.write('NULL,') |
| 134 continue |
| 135 outfile.write('%.4g,' % func(self.cols[fullconfig])) |
118 outfile.write('\n') | 136 outfile.write('\n') |
119 | 137 |
120 | |
121 def main(): | 138 def main(): |
122 parser = Parser() | 139 parser = Parser() |
123 | 140 |
124 # Parse the input files. | 141 # Parse the input files. |
125 for src in FLAGS.sources: | 142 for src in FLAGS.sources: |
126 if src == '-': | 143 if src == '-': |
127 parser.parse_file(sys.stdin) | 144 parser.parse_file(sys.stdin) |
128 else: | 145 else: |
129 with open(src, mode='r') as infile: | 146 with open(src, mode='r') as infile: |
130 parser.parse_file(infile) | 147 parser.parse_file(infile) |
131 | 148 |
132 # Print the csv. | 149 # Print the csv. |
133 if not FLAGS.open: | 150 if not FLAGS.open: |
134 parser.print_csv() | 151 parser.print_csv() |
135 else: | 152 else: |
136 dirname = tempfile.mkdtemp() | 153 dirname = tempfile.mkdtemp() |
137 basename = FLAGS.name | 154 basename = FLAGS.name |
138 if os.path.splitext(basename)[1] != '.csv': | 155 if os.path.splitext(basename)[1] != '.csv': |
139 basename += '.csv'; | 156 basename += '.csv'; |
140 pathname = os.path.join(dirname, basename) | 157 pathname = os.path.join(dirname, basename) |
141 with open(pathname, mode='w') as tmpfile: | 158 with open(pathname, mode='w') as tmpfile: |
142 parser.print_csv(outfile=tmpfile) | 159 parser.print_csv(outfile=tmpfile) |
143 fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) | 160 fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) |
144 print('opening %s' % fileuri) | 161 print('opening %s' % fileuri) |
145 webbrowser.open(fileuri) | 162 webbrowser.open(fileuri) |
146 | 163 |
147 | 164 |
148 if __name__ == '__main__': | 165 if __name__ == '__main__': |
149 main() | 166 main() |
OLD | NEW |