| OLD | NEW |
| 1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 64 self._host = self._port.host | 64 self._host = self._port.host |
| 65 else: | 65 else: |
| 66 self._host = Host() | 66 self._host = Host() |
| 67 self._port = self._host.port_factory.get(self._options.platform, sel
f._options) | 67 self._port = self._host.port_factory.get(self._options.platform, sel
f._options) |
| 68 self._host.initialize_scm() | 68 self._host.initialize_scm() |
| 69 self._webkit_base_dir_len = len(self._port.webkit_base()) | 69 self._webkit_base_dir_len = len(self._port.webkit_base()) |
| 70 self._base_path = self._port.perf_tests_dir() | 70 self._base_path = self._port.perf_tests_dir() |
| 71 self._timestamp = time.time() | 71 self._timestamp = time.time() |
| 72 self._utc_timestamp = datetime.datetime.utcnow() | 72 self._utc_timestamp = datetime.datetime.utcnow() |
| 73 | 73 |
| 74 | |
| 75 @staticmethod | 74 @staticmethod |
| 76 def _parse_args(args=None): | 75 def _parse_args(args=None): |
| 77 def _expand_path(option, opt_str, value, parser): | 76 def _expand_path(option, opt_str, value, parser): |
| 78 path = os.path.expandvars(os.path.expanduser(value)) | 77 path = os.path.expandvars(os.path.expanduser(value)) |
| 79 setattr(parser.values, option.dest, path) | 78 setattr(parser.values, option.dest, path) |
| 80 perf_option_list = [ | 79 perf_option_list = [ |
| 81 optparse.make_option('--debug', action='store_const', const='Debug',
dest="configuration", | 80 optparse.make_option('--debug', action='store_const', const='Debug',
dest="configuration", |
| 82 help='Set the configuration to Debug'), | 81 help='Set the configuration to Debug'), |
| 83 optparse.make_option('--release', action='store_const', const='Relea
se', dest="configuration", | 82 optparse.make_option('--release', action='store_const', const='Relea
se', dest="configuration", |
| 84 help='Set the configuration to Release'), | 83 help='Set the configuration to Release'), |
| 85 optparse.make_option("--platform", | 84 optparse.make_option("--platform", |
| 86 help="Specify port/platform being tested (e.g. mac)"), | 85 help="Specify port/platform being tested (e.g.
mac)"), |
| 87 optparse.make_option("--chromium", | 86 optparse.make_option("--chromium", |
| 88 action="store_const", const='chromium', dest='platform', help='A
lias for --platform=chromium'), | 87 action="store_const", const='chromium', dest='p
latform', help='Alias for --platform=chromium'), |
| 89 optparse.make_option("--android", | 88 optparse.make_option("--android", |
| 90 action="store_const", const='android', dest='platform', help='Al
ias for --platform=android'), | 89 action="store_const", const='android', dest='pl
atform', help='Alias for --platform=android'), |
| 91 optparse.make_option("--builder-name", | 90 optparse.make_option("--builder-name", |
| 92 help=("The name of the builder shown on the waterfall running th
is script e.g. google-mac-2.")), | 91 help=("The name of the builder shown on the wat
erfall running this script e.g. google-mac-2.")), |
| 93 optparse.make_option("--build-number", | 92 optparse.make_option("--build-number", |
| 94 help=("The build number of the builder running this script.")), | 93 help=("The build number of the builder running
this script.")), |
| 95 optparse.make_option("--build", dest="build", action="store_true", d
efault=True, | 94 optparse.make_option("--build", dest="build", action="store_true", d
efault=True, |
| 96 help="Check to ensure the DumpRenderTree build is up-to-date (de
fault)."), | 95 help="Check to ensure the DumpRenderTree build
is up-to-date (default)."), |
| 97 optparse.make_option("--no-build", dest="build", action="store_false
", | 96 optparse.make_option("--no-build", dest="build", action="store_false
", |
| 98 help="Don't check to see if the DumpRenderTree build is up-to-da
te."), | 97 help="Don't check to see if the DumpRenderTree
build is up-to-date."), |
| 99 optparse.make_option("--build-directory", | 98 optparse.make_option("--build-directory", |
| 100 help="Path to the directory under which build files are kept (sh
ould not include configuration)"), | 99 help="Path to the directory under which build f
iles are kept (should not include configuration)"), |
| 101 optparse.make_option("--time-out-ms", default=600 * 1000, | 100 optparse.make_option("--time-out-ms", default=600 * 1000, |
| 102 help="Set the timeout for each test"), | 101 help="Set the timeout for each test"), |
| 103 optparse.make_option("--no-results", action="store_false", dest="gen
erate_results", default=True, | 102 optparse.make_option("--no-results", action="store_false", dest="gen
erate_results", default=True, |
| 104 help="Do no generate results JSON and results page."), | 103 help="Do no generate results JSON and results p
age."), |
| 105 optparse.make_option("--output-json-path", action='callback', callba
ck=_expand_path, type="str", | 104 optparse.make_option("--output-json-path", action='callback', callba
ck=_expand_path, type="str", |
| 106 help="Path to generate a JSON file at; may contain previous resu
lts if it already exists."), | 105 help="Path to generate a JSON file at; may cont
ain previous results if it already exists."), |
| 107 optparse.make_option("--reset-results", action="store_true", | 106 optparse.make_option("--reset-results", action="store_true", |
| 108 help="Clears the content in the generated JSON file before addin
g the results."), | 107 help="Clears the content in the generated JSON
file before adding the results."), |
| 109 optparse.make_option("--slave-config-json-path", action='callback',
callback=_expand_path, type="str", | 108 optparse.make_option("--slave-config-json-path", action='callback',
callback=_expand_path, type="str", |
| 110 help="Only used on bots. Path to a slave configuration file."), | 109 help="Only used on bots. Path to a slave config
uration file."), |
| 111 optparse.make_option("--description", | 110 optparse.make_option("--description", |
| 112 help="Add a description to the output JSON file if one is genera
ted"), | 111 help="Add a description to the output JSON file
if one is generated"), |
| 113 optparse.make_option("--no-show-results", action="store_false", defa
ult=True, dest="show_results", | 112 optparse.make_option("--no-show-results", action="store_false", defa
ult=True, dest="show_results", |
| 114 help="Don't launch a browser with results after the tests are do
ne"), | 113 help="Don't launch a browser with results after
the tests are done"), |
| 115 optparse.make_option("--test-results-server", | 114 optparse.make_option("--test-results-server", |
| 116 help="Upload the generated JSON file to the specified server whe
n --output-json-path is present."), | 115 help="Upload the generated JSON file to the spe
cified server when --output-json-path is present."), |
| 117 optparse.make_option("--force", dest="use_skipped_list", action="sto
re_false", default=True, | 116 optparse.make_option("--force", dest="use_skipped_list", action="sto
re_false", default=True, |
| 118 help="Run all tests, including the ones in the Skipped list."), | 117 help="Run all tests, including the ones in the
Skipped list."), |
| 119 optparse.make_option("--profile", action="store_true", | 118 optparse.make_option("--profile", action="store_true", |
| 120 help="Output per-test profile information."), | 119 help="Output per-test profile information."), |
| 121 optparse.make_option("--profiler", action="store", | 120 optparse.make_option("--profiler", action="store", |
| 122 help="Output per-test profile information, using the specified p
rofiler."), | 121 help="Output per-test profile information, usin
g the specified profiler."), |
| 123 optparse.make_option("--additional-driver-flag", action="append", | 122 optparse.make_option("--additional-driver-flag", action="append", |
| 124 default=[], help="Additional command line flag to pass to DumpRe
nderTree " | 123 default=[], help="Additional command line flag
to pass to DumpRenderTree " |
| 125 "Specify multiple times to add multiple flags."), | 124 "Specify multiple times to add multiple flags."
), |
| 126 optparse.make_option("--driver-name", type="string", | 125 optparse.make_option("--driver-name", type="string", |
| 127 help="Alternative DumpRenderTree binary to use"), | 126 help="Alternative DumpRenderTree binary to use"
), |
| 128 optparse.make_option("--content-shell", action="store_true", | 127 optparse.make_option("--content-shell", action="store_true", |
| 129 help="Use Content Shell instead of DumpRenderTree"), | 128 help="Use Content Shell instead of DumpRenderTr
ee"), |
| 130 optparse.make_option("--repeat", default=1, type="int", | 129 optparse.make_option("--repeat", default=1, type="int", |
| 131 help="Specify number of times to run test set (default: 1)."), | 130 help="Specify number of times to run test set (
default: 1)."), |
| 132 optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUN
NER_COUNT, type="int", | 131 optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUN
NER_COUNT, type="int", |
| 133 help="Specify number of times to invoke test runner for each per
formance test."), | 132 help="Specify number of times to invoke test ru
nner for each performance test."), |
| 134 ] | 133 ] |
| 135 return optparse.OptionParser(option_list=(perf_option_list)).parse_args(
args) | 134 return optparse.OptionParser(option_list=(perf_option_list)).parse_args(
args) |
| 136 | 135 |
| 137 def _collect_tests(self): | 136 def _collect_tests(self): |
| 138 test_extensions = ['.html', '.svg'] | 137 test_extensions = ['.html', '.svg'] |
| 139 | 138 |
| 140 def _is_test_file(filesystem, dirname, filename): | 139 def _is_test_file(filesystem, dirname, filename): |
| 141 return filesystem.splitext(filename)[1] in test_extensions | 140 return filesystem.splitext(filename)[1] in test_extensions |
| 142 | 141 |
| 143 filesystem = self._host.filesystem | 142 filesystem = self._host.filesystem |
| 144 | 143 |
| 145 paths = [] | 144 paths = [] |
| 146 for arg in self._args: | 145 for arg in self._args: |
| 147 if filesystem.exists(filesystem.join(self._base_path, arg)): | 146 if filesystem.exists(filesystem.join(self._base_path, arg)): |
| 148 paths.append(arg) | 147 paths.append(arg) |
| 149 else: | 148 else: |
| 150 relpath = filesystem.relpath(arg, self._base_path) | 149 relpath = filesystem.relpath(arg, self._base_path) |
| 151 if filesystem.exists(filesystem.join(self._base_path, relpath)): | 150 if filesystem.exists(filesystem.join(self._base_path, relpath)): |
| 152 paths.append(filesystem.normpath(relpath)) | 151 paths.append(filesystem.normpath(relpath)) |
| 153 else: | 152 else: |
| 154 _log.warn('Path was not found:' + arg) | 153 _log.warn('Path was not found:' + arg) |
| 155 | 154 |
| 156 skipped_directories = set(['.svn', 'resources']) | 155 skipped_directories = set(['.svn', 'resources']) |
| 157 test_files = find_files.find(filesystem, self._base_path, paths, skipped
_directories, _is_test_file) | 156 test_files = find_files.find(filesystem, self._base_path, paths, skipped
_directories, _is_test_file) |
| 158 tests = [] | 157 tests = [] |
| 159 for path in test_files: | 158 for path in test_files: |
| 160 relative_path = filesystem.relpath(path, self._base_path).replace('\
\', '/') | 159 relative_path = filesystem.relpath(path, self._base_path).replace('\
\', '/') |
| 161 if self._options.use_skipped_list and self._port.skips_perf_test(rel
ative_path) and filesystem.normpath(relative_path) not in paths: | 160 if self._options.use_skipped_list and self._port.skips_perf_test(rel
ative_path) and filesystem.normpath(relative_path) not in paths: |
| 162 continue | 161 continue |
| 163 test = PerfTestFactory.create_perf_test(self._port, relative_path, p
ath, test_runner_count=self._options.test_runner_count) | 162 test = PerfTestFactory.create_perf_test(self._port, relative_path, p
ath, |
| 163 test_runner_count=self._opti
ons.test_runner_count) |
| 164 tests.append(test) | 164 tests.append(test) |
| 165 | 165 |
| 166 return tests | 166 return tests |
| 167 | 167 |
| 168 def _start_http_servers(self): | 168 def _start_http_servers(self): |
| 169 self._port.acquire_http_lock() | 169 self._port.acquire_http_lock() |
| 170 self._port.start_http_server(number_of_servers=2) | 170 self._port.start_http_server(number_of_servers=2) |
| 171 | 171 |
| 172 def _stop_http_servers(self): | 172 def _stop_http_servers(self): |
| 173 self._port.stop_http_server() | 173 self._port.stop_http_server() |
| 174 self._port.release_http_lock() | 174 self._port.release_http_lock() |
| 175 | 175 |
| 176 def run(self): | 176 def run(self): |
| 177 needs_http = self._port.requires_http_server() | 177 needs_http = self._port.requires_http_server() |
| 178 | 178 |
| 179 class FakePrinter(object): | 179 class FakePrinter(object): |
| 180 |
| 180 def write_update(self, msg): | 181 def write_update(self, msg): |
| 181 print msg | 182 print msg |
| 182 | 183 |
| 183 def write_throttled_update(self, msg): | 184 def write_throttled_update(self, msg): |
| 184 pass | 185 pass |
| 185 | 186 |
| 186 if self._port.check_build(needs_http=needs_http, printer=FakePrinter()): | 187 if self._port.check_build(needs_http=needs_http, printer=FakePrinter()): |
| 187 _log.error("Build not up to date for %s" % self._port._path_to_drive
r()) | 188 _log.error("Build not up to date for %s" % self._port._path_to_drive
r()) |
| 188 return self.EXIT_CODE_BAD_BUILD | 189 return self.EXIT_CODE_BAD_BUILD |
| 189 | 190 |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 229 if output_json_path: | 230 if output_json_path: |
| 230 return output_json_path | 231 return output_json_path |
| 231 return self._host.filesystem.join(self._port.perf_results_directory(), s
elf._DEFAULT_JSON_FILENAME) | 232 return self._host.filesystem.join(self._port.perf_results_directory(), s
elf._DEFAULT_JSON_FILENAME) |
| 232 | 233 |
| 233 def _results_page_path(self): | 234 def _results_page_path(self): |
| 234 return self._host.filesystem.splitext(self._output_json_path())[0] + '.h
tml' | 235 return self._host.filesystem.splitext(self._output_json_path())[0] + '.h
tml' |
| 235 | 236 |
| 236 def _generate_results(self): | 237 def _generate_results(self): |
| 237 options = self._options | 238 options = self._options |
| 238 output_json_path = self._output_json_path() | 239 output_json_path = self._output_json_path() |
| 239 output = self._generate_results_dict(self._timestamp, options.descriptio
n, options.platform, options.builder_name, options.build_number) | 240 output = self._generate_results_dict(self._timestamp, options.descriptio
n, |
| 241 options.platform, options.builder_n
ame, options.build_number) |
| 240 | 242 |
| 241 if options.slave_config_json_path: | 243 if options.slave_config_json_path: |
| 242 output = self._merge_slave_config_json(options.slave_config_json_pat
h, output) | 244 output = self._merge_slave_config_json(options.slave_config_json_pat
h, output) |
| 243 if not output: | 245 if not output: |
| 244 return self.EXIT_CODE_BAD_SOURCE_JSON | 246 return self.EXIT_CODE_BAD_SOURCE_JSON |
| 245 | 247 |
| 246 output = self._merge_outputs_if_needed(output_json_path, output) | 248 output = self._merge_outputs_if_needed(output_json_path, output) |
| 247 if not output: | 249 if not output: |
| 248 return self.EXIT_CODE_BAD_MERGE | 250 return self.EXIT_CODE_BAD_MERGE |
| 249 | 251 |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 369 if metrics: | 371 if metrics: |
| 370 self._results.append((test, metrics)) | 372 self._results.append((test, metrics)) |
| 371 else: | 373 else: |
| 372 failures += 1 | 374 failures += 1 |
| 373 _log.error('FAILED') | 375 _log.error('FAILED') |
| 374 | 376 |
| 375 _log.info('Finished: %f s' % (time.time() - start_time)) | 377 _log.info('Finished: %f s' % (time.time() - start_time)) |
| 376 _log.info('') | 378 _log.info('') |
| 377 | 379 |
| 378 return failures | 380 return failures |
| OLD | NEW |