Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(99)

Side by Side Diff: mojo/devtools/common/mojo_benchmark

Issue 1407273007: mojo_benchmark: don't duplicate benchmark name in variant name. (Closed) Base URL: git@github.com:domokit/mojo.git@master
Patch Set: Created 5 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Copyright 2015 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Runner for Mojo application benchmarks.""" 6 """Runner for Mojo application benchmarks."""
7 7
8 import argparse 8 import argparse
9 import json 9 import json
10 import logging 10 import logging
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
76 76
77 def _generate_benchmark_variants(benchmark_spec): 77 def _generate_benchmark_variants(benchmark_spec):
78 """Generates benchmark specifications for individual variants of the given 78 """Generates benchmark specifications for individual variants of the given
79 benchmark: cold start and warm start. 79 benchmark: cold start and warm start.
80 80
81 Returns: 81 Returns:
82 A list of benchmark specs corresponding to individual variants of the given 82 A list of benchmark specs corresponding to individual variants of the given
83 benchmark. 83 benchmark.
84 """ 84 """
85 variants = [] 85 variants = []
86 # Cold start.
87 variants.append({ 86 variants.append({
88 'name': benchmark_spec['name'] + ' (cold start)', 87 'variant_name': 'cold start',
89 'app': benchmark_spec['app'], 88 'app': benchmark_spec['app'],
90 'duration': benchmark_spec['duration'], 89 'duration': benchmark_spec['duration'],
91 'measurements': benchmark_spec['measurements'], 90 'measurements': benchmark_spec['measurements'],
92 'shell-args': benchmark_spec.get('shell-args', 91 'shell-args': benchmark_spec.get('shell-args',
93 []) + _COLD_START_SHELL_ARGS}) 92 []) + _COLD_START_SHELL_ARGS})
94 # Warm start.
95 variants.append({ 93 variants.append({
96 'name': benchmark_spec['name'] + ' (warm start)', 94 'variant_name': 'warm start',
97 'app': benchmark_spec['app'], 95 'app': benchmark_spec['app'],
98 'duration': benchmark_spec['duration'], 96 'duration': benchmark_spec['duration'],
99 'measurements': benchmark_spec['measurements'], 97 'measurements': benchmark_spec['measurements'],
100 'shell-args': benchmark_spec.get('shell-args', [])}) 98 'shell-args': benchmark_spec.get('shell-args', [])})
101 return variants 99 return variants
102 100
103 101
104 def _run_benchmark(shell, shell_args, name, app, duration_seconds, measurements, 102 def _run_benchmark(shell, shell_args, name, app, duration_seconds, measurements,
105 verbose, android, save_traces): 103 verbose, android, output_file):
106 """Runs the given benchmark by running `benchmark.mojo` in mojo shell with 104 """Runs the given benchmark by running `benchmark.mojo` in mojo shell with
107 appropriate arguments and returns the produced output. 105 appropriate arguments and returns the produced output.
108 106
109 Returns: 107 Returns:
110 A tuple of (succeeded, error_msg, output). 108 A tuple of (succeeded, error_msg, output).
111 """ 109 """
112 timeout = duration_seconds + _EXTRA_TIMEOUT 110 timeout = duration_seconds + _EXTRA_TIMEOUT
113 benchmark_args = [] 111 benchmark_args = []
114 benchmark_args.append('--app=' + app) 112 benchmark_args.append('--app=' + app)
115 benchmark_args.append('--duration=' + str(duration_seconds)) 113 benchmark_args.append('--duration=' + str(duration_seconds))
116 114
117 output_file = None
118 device_output_file = None 115 device_output_file = None
119 if save_traces: 116 if output_file:
120 output_file = 'benchmark-%s-%s.trace' % (name.replace(' ', '_'),
121 time.strftime('%Y%m%d%H%M%S'))
122 if android: 117 if android:
123 device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file) 118 device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file)
124 benchmark_args.append('--trace-output=' + device_output_file) 119 benchmark_args.append('--trace-output=' + device_output_file)
125 else: 120 else:
126 benchmark_args.append('--trace-output=' + output_file) 121 benchmark_args.append('--trace-output=' + output_file)
127 122
128 for measurement in measurements: 123 for measurement in measurements:
129 benchmark_args.append(measurement) 124 benchmark_args.append(measurement)
130 125
131 shell_args = list(shell_args) 126 shell_args = list(shell_args)
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
191 except shell_arguments.ShellConfigurationException as e: 186 except shell_arguments.ShellConfigurationException as e:
192 print e 187 print e
193 return 1 188 return 1
194 189
195 target_os = 'android' if script_args.android else 'linux' 190 target_os = 'android' if script_args.android else 'linux'
196 benchmark_list_params = {"target_os": target_os} 191 benchmark_list_params = {"target_os": target_os}
197 exec script_args.benchmark_list_file in benchmark_list_params 192 exec script_args.benchmark_list_file in benchmark_list_params
198 193
199 exit_code = 0 194 exit_code = 0
200 for benchmark_spec in benchmark_list_params['benchmarks']: 195 for benchmark_spec in benchmark_list_params['benchmarks']:
196 benchmark_name = benchmark_spec['name']
197 chart_data_recorder = None
198 if script_args.chart_data_output_file:
199 chart_data_recorder = perf_dashboard.ChartDataRecorder(benchmark_name)
200
201 for variant_spec in _generate_benchmark_variants(benchmark_spec): 201 for variant_spec in _generate_benchmark_variants(benchmark_spec):
202 name = variant_spec['name'] 202 variant_name = variant_spec['variant_name']
203 app = variant_spec['app'] 203 app = variant_spec['app']
204 duration = variant_spec['duration'] 204 duration = variant_spec['duration']
205 shell_args = variant_spec.get('shell-args', []) + common_shell_args 205 shell_args = variant_spec.get('shell-args', []) + common_shell_args
206 measurements = variant_spec['measurements'] 206 measurements = variant_spec['measurements']
207 207
208 chart_data_recorder = None 208 output_file = None
209 if script_args.chart_data_output_file: 209 if script_args.save_traces:
210 chart_data_recorder = perf_dashboard.ChartDataRecorder(name) 210 output_file = 'benchmark-%s-%s-%s.trace' % (
211 benchmark_name.replace(' ', '_'),
212 variant_name.replace(' ', '_'),
213 time.strftime('%Y%m%d%H%M%S'))
211 benchmark_succeeded, benchmark_error, output = _run_benchmark( 214 benchmark_succeeded, benchmark_error, output = _run_benchmark(
212 shell, shell_args, name, app, duration, measurements, 215 shell, shell_args, variant_name, app, duration, measurements,
213 script_args.verbose, script_args.android, 216 script_args.verbose, script_args.android, output_file)
214 script_args.save_traces)
215 217
216 print '[ %s ]' % name 218 print '[ %s ] %s ' % (benchmark_name, variant_name)
217 219
218 some_measurements_failed = False 220 some_measurements_failed = False
219 if benchmark_succeeded: 221 if benchmark_succeeded:
220 measurement_results = _parse_measurement_results(output) 222 measurement_results = _parse_measurement_results(output)
221 # Iterate over the list of specs, not the dictionary, to detect missing 223 # Iterate over the list of specs, not the dictionary, to detect missing
222 # results and preserve the required order. 224 # results and preserve the required order.
223 for measurement_spec in measurements: 225 for measurement_spec in measurements:
224 if measurement_spec in measurement_results: 226 if measurement_spec in measurement_results:
225 result = measurement_results[measurement_spec] 227 result = measurement_results[measurement_spec]
226 print '%s %s' % (measurement_spec, result) 228 print '%s %s' % (measurement_spec, result)
227 229
228 if chart_data_recorder: 230 if chart_data_recorder:
229 measurement_name = measurement_spec.replace('/', '-') 231 measurement_name = measurement_spec.replace('/', '-')
230 chart_data_recorder.record_scalar(name, measurement_name, 'ms', 232 chart_data_recorder.record_scalar(variant_name, measurement_name,
231 result) 233 'ms', result)
232 else: 234 else:
233 print '%s ?' % measurement_spec 235 print '%s ?' % measurement_spec
234 some_measurements_failed = True 236 some_measurements_failed = True
235 237
236 if not benchmark_succeeded or some_measurements_failed: 238 if not benchmark_succeeded or some_measurements_failed:
237 if not benchmark_succeeded: 239 if not benchmark_succeeded:
238 print 'benchmark failed: ' + benchmark_error 240 print 'benchmark failed: ' + benchmark_error
239 if some_measurements_failed: 241 if some_measurements_failed:
240 print 'some measurements failed' 242 print 'some measurements failed'
241 print 'output: ' 243 print 'output: '
242 print '-' * 72 244 print '-' * 72
243 print output 245 print output
244 print '-' * 72 246 print '-' * 72
245 exit_code = 1 247 exit_code = 1
246 248
247 if script_args.chart_data_output_file: 249 if script_args.chart_data_output_file:
248 script_args.chart_data_output_file.write( 250 script_args.chart_data_output_file.write(
249 json.dumps(chart_data_recorder.get_chart_data())) 251 json.dumps(chart_data_recorder.get_chart_data()))
250 script_args.chart_data_output_file.write('\n') 252 script_args.chart_data_output_file.write('\n')
251 253
252 return exit_code 254 return exit_code
253 255
254 if __name__ == '__main__': 256 if __name__ == '__main__':
255 sys.exit(main()) 257 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698