| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2015 The Chromium Authors. All rights reserved. | 2 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Runner for Mojo application benchmarks.""" | 6 """Runner for Mojo application benchmarks.""" |
| 7 | 7 |
| 8 import argparse | 8 import argparse |
| 9 import json | 9 import json |
| 10 import logging | 10 import logging |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 76 | 76 |
| 77 def _generate_benchmark_variants(benchmark_spec): | 77 def _generate_benchmark_variants(benchmark_spec): |
| 78 """Generates benchmark specifications for individual variants of the given | 78 """Generates benchmark specifications for individual variants of the given |
| 79 benchmark: cold start and warm start. | 79 benchmark: cold start and warm start. |
| 80 | 80 |
| 81 Returns: | 81 Returns: |
| 82 A list of benchmark specs corresponding to individual variants of the given | 82 A list of benchmark specs corresponding to individual variants of the given |
| 83 benchmark. | 83 benchmark. |
| 84 """ | 84 """ |
| 85 variants = [] | 85 variants = [] |
| 86 # Cold start. | |
| 87 variants.append({ | 86 variants.append({ |
| 88 'name': benchmark_spec['name'] + ' (cold start)', | 87 'variant_name': 'cold start', |
| 89 'app': benchmark_spec['app'], | 88 'app': benchmark_spec['app'], |
| 90 'duration': benchmark_spec['duration'], | 89 'duration': benchmark_spec['duration'], |
| 91 'measurements': benchmark_spec['measurements'], | 90 'measurements': benchmark_spec['measurements'], |
| 92 'shell-args': benchmark_spec.get('shell-args', | 91 'shell-args': benchmark_spec.get('shell-args', |
| 93 []) + _COLD_START_SHELL_ARGS}) | 92 []) + _COLD_START_SHELL_ARGS}) |
| 94 # Warm start. | |
| 95 variants.append({ | 93 variants.append({ |
| 96 'name': benchmark_spec['name'] + ' (warm start)', | 94 'variant_name': 'warm start', |
| 97 'app': benchmark_spec['app'], | 95 'app': benchmark_spec['app'], |
| 98 'duration': benchmark_spec['duration'], | 96 'duration': benchmark_spec['duration'], |
| 99 'measurements': benchmark_spec['measurements'], | 97 'measurements': benchmark_spec['measurements'], |
| 100 'shell-args': benchmark_spec.get('shell-args', [])}) | 98 'shell-args': benchmark_spec.get('shell-args', [])}) |
| 101 return variants | 99 return variants |
| 102 | 100 |
| 103 | 101 |
| 104 def _run_benchmark(shell, shell_args, name, app, duration_seconds, measurements, | 102 def _run_benchmark(shell, shell_args, name, app, duration_seconds, measurements, |
| 105 verbose, android, save_traces): | 103 verbose, android, output_file): |
| 106 """Runs the given benchmark by running `benchmark.mojo` in mojo shell with | 104 """Runs the given benchmark by running `benchmark.mojo` in mojo shell with |
| 107 appropriate arguments and returns the produced output. | 105 appropriate arguments and returns the produced output. |
| 108 | 106 |
| 109 Returns: | 107 Returns: |
| 110 A tuple of (succeeded, error_msg, output). | 108 A tuple of (succeeded, error_msg, output). |
| 111 """ | 109 """ |
| 112 timeout = duration_seconds + _EXTRA_TIMEOUT | 110 timeout = duration_seconds + _EXTRA_TIMEOUT |
| 113 benchmark_args = [] | 111 benchmark_args = [] |
| 114 benchmark_args.append('--app=' + app) | 112 benchmark_args.append('--app=' + app) |
| 115 benchmark_args.append('--duration=' + str(duration_seconds)) | 113 benchmark_args.append('--duration=' + str(duration_seconds)) |
| 116 | 114 |
| 117 output_file = None | |
| 118 device_output_file = None | 115 device_output_file = None |
| 119 if save_traces: | 116 if output_file: |
| 120 output_file = 'benchmark-%s-%s.trace' % (name.replace(' ', '_'), | |
| 121 time.strftime('%Y%m%d%H%M%S')) | |
| 122 if android: | 117 if android: |
| 123 device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file) | 118 device_output_file = os.path.join(shell.get_tmp_dir_path(), output_file) |
| 124 benchmark_args.append('--trace-output=' + device_output_file) | 119 benchmark_args.append('--trace-output=' + device_output_file) |
| 125 else: | 120 else: |
| 126 benchmark_args.append('--trace-output=' + output_file) | 121 benchmark_args.append('--trace-output=' + output_file) |
| 127 | 122 |
| 128 for measurement in measurements: | 123 for measurement in measurements: |
| 129 benchmark_args.append(measurement) | 124 benchmark_args.append(measurement) |
| 130 | 125 |
| 131 shell_args = list(shell_args) | 126 shell_args = list(shell_args) |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 191 except shell_arguments.ShellConfigurationException as e: | 186 except shell_arguments.ShellConfigurationException as e: |
| 192 print e | 187 print e |
| 193 return 1 | 188 return 1 |
| 194 | 189 |
| 195 target_os = 'android' if script_args.android else 'linux' | 190 target_os = 'android' if script_args.android else 'linux' |
| 196 benchmark_list_params = {"target_os": target_os} | 191 benchmark_list_params = {"target_os": target_os} |
| 197 exec script_args.benchmark_list_file in benchmark_list_params | 192 exec script_args.benchmark_list_file in benchmark_list_params |
| 198 | 193 |
| 199 exit_code = 0 | 194 exit_code = 0 |
| 200 for benchmark_spec in benchmark_list_params['benchmarks']: | 195 for benchmark_spec in benchmark_list_params['benchmarks']: |
| 196 benchmark_name = benchmark_spec['name'] |
| 197 chart_data_recorder = None |
| 198 if script_args.chart_data_output_file: |
| 199 chart_data_recorder = perf_dashboard.ChartDataRecorder(benchmark_name) |
| 200 |
| 201 for variant_spec in _generate_benchmark_variants(benchmark_spec): | 201 for variant_spec in _generate_benchmark_variants(benchmark_spec): |
| 202 name = variant_spec['name'] | 202 variant_name = variant_spec['variant_name'] |
| 203 app = variant_spec['app'] | 203 app = variant_spec['app'] |
| 204 duration = variant_spec['duration'] | 204 duration = variant_spec['duration'] |
| 205 shell_args = variant_spec.get('shell-args', []) + common_shell_args | 205 shell_args = variant_spec.get('shell-args', []) + common_shell_args |
| 206 measurements = variant_spec['measurements'] | 206 measurements = variant_spec['measurements'] |
| 207 | 207 |
| 208 chart_data_recorder = None | 208 output_file = None |
| 209 if script_args.chart_data_output_file: | 209 if script_args.save_traces: |
| 210 chart_data_recorder = perf_dashboard.ChartDataRecorder(name) | 210 output_file = 'benchmark-%s-%s-%s.trace' % ( |
| 211 benchmark_name.replace(' ', '_'), |
| 212 variant_name.replace(' ', '_'), |
| 213 time.strftime('%Y%m%d%H%M%S')) |
| 211 benchmark_succeeded, benchmark_error, output = _run_benchmark( | 214 benchmark_succeeded, benchmark_error, output = _run_benchmark( |
| 212 shell, shell_args, name, app, duration, measurements, | 215 shell, shell_args, variant_name, app, duration, measurements, |
| 213 script_args.verbose, script_args.android, | 216 script_args.verbose, script_args.android, output_file) |
| 214 script_args.save_traces) | |
| 215 | 217 |
| 216 print '[ %s ]' % name | 218 print '[ %s ] %s ' % (benchmark_name, variant_name) |
| 217 | 219 |
| 218 some_measurements_failed = False | 220 some_measurements_failed = False |
| 219 if benchmark_succeeded: | 221 if benchmark_succeeded: |
| 220 measurement_results = _parse_measurement_results(output) | 222 measurement_results = _parse_measurement_results(output) |
| 221 # Iterate over the list of specs, not the dictionary, to detect missing | 223 # Iterate over the list of specs, not the dictionary, to detect missing |
| 222 # results and preserve the required order. | 224 # results and preserve the required order. |
| 223 for measurement_spec in measurements: | 225 for measurement_spec in measurements: |
| 224 if measurement_spec in measurement_results: | 226 if measurement_spec in measurement_results: |
| 225 result = measurement_results[measurement_spec] | 227 result = measurement_results[measurement_spec] |
| 226 print '%s %s' % (measurement_spec, result) | 228 print '%s %s' % (measurement_spec, result) |
| 227 | 229 |
| 228 if chart_data_recorder: | 230 if chart_data_recorder: |
| 229 measurement_name = measurement_spec.replace('/', '-') | 231 measurement_name = measurement_spec.replace('/', '-') |
| 230 chart_data_recorder.record_scalar(name, measurement_name, 'ms', | 232 chart_data_recorder.record_scalar(variant_name, measurement_name, |
| 231 result) | 233 'ms', result) |
| 232 else: | 234 else: |
| 233 print '%s ?' % measurement_spec | 235 print '%s ?' % measurement_spec |
| 234 some_measurements_failed = True | 236 some_measurements_failed = True |
| 235 | 237 |
| 236 if not benchmark_succeeded or some_measurements_failed: | 238 if not benchmark_succeeded or some_measurements_failed: |
| 237 if not benchmark_succeeded: | 239 if not benchmark_succeeded: |
| 238 print 'benchmark failed: ' + benchmark_error | 240 print 'benchmark failed: ' + benchmark_error |
| 239 if some_measurements_failed: | 241 if some_measurements_failed: |
| 240 print 'some measurements failed' | 242 print 'some measurements failed' |
| 241 print 'output: ' | 243 print 'output: ' |
| 242 print '-' * 72 | 244 print '-' * 72 |
| 243 print output | 245 print output |
| 244 print '-' * 72 | 246 print '-' * 72 |
| 245 exit_code = 1 | 247 exit_code = 1 |
| 246 | 248 |
| 247 if script_args.chart_data_output_file: | 249 if script_args.chart_data_output_file: |
| 248 script_args.chart_data_output_file.write( | 250 script_args.chart_data_output_file.write( |
| 249 json.dumps(chart_data_recorder.get_chart_data())) | 251 json.dumps(chart_data_recorder.get_chart_data())) |
| 250 script_args.chart_data_output_file.write('\n') | 252 script_args.chart_data_output_file.write('\n') |
| 251 | 253 |
| 252 return exit_code | 254 return exit_code |
| 253 | 255 |
| 254 if __name__ == '__main__': | 256 if __name__ == '__main__': |
| 255 sys.exit(main()) | 257 sys.exit(main()) |
| OLD | NEW |