OLD | NEW |
1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
64 self._host = self._port.host | 64 self._host = self._port.host |
65 else: | 65 else: |
66 self._host = Host() | 66 self._host = Host() |
67 self._port = self._host.port_factory.get(self._options.platform, sel
f._options) | 67 self._port = self._host.port_factory.get(self._options.platform, sel
f._options) |
68 self._host.initialize_scm() | 68 self._host.initialize_scm() |
69 self._webkit_base_dir_len = len(self._port.webkit_base()) | 69 self._webkit_base_dir_len = len(self._port.webkit_base()) |
70 self._base_path = self._port.perf_tests_dir() | 70 self._base_path = self._port.perf_tests_dir() |
71 self._timestamp = time.time() | 71 self._timestamp = time.time() |
72 self._utc_timestamp = datetime.datetime.utcnow() | 72 self._utc_timestamp = datetime.datetime.utcnow() |
73 | 73 |
74 | |
75 @staticmethod | 74 @staticmethod |
76 def _parse_args(args=None): | 75 def _parse_args(args=None): |
77 def _expand_path(option, opt_str, value, parser): | 76 def _expand_path(option, opt_str, value, parser): |
78 path = os.path.expandvars(os.path.expanduser(value)) | 77 path = os.path.expandvars(os.path.expanduser(value)) |
79 setattr(parser.values, option.dest, path) | 78 setattr(parser.values, option.dest, path) |
80 perf_option_list = [ | 79 perf_option_list = [ |
81 optparse.make_option('--debug', action='store_const', const='Debug',
dest="configuration", | 80 optparse.make_option('--debug', action='store_const', const='Debug',
dest="configuration", |
82 help='Set the configuration to Debug'), | 81 help='Set the configuration to Debug'), |
83 optparse.make_option('--release', action='store_const', const='Relea
se', dest="configuration", | 82 optparse.make_option('--release', action='store_const', const='Relea
se', dest="configuration", |
84 help='Set the configuration to Release'), | 83 help='Set the configuration to Release'), |
85 optparse.make_option("--platform", | 84 optparse.make_option("--platform", |
86 help="Specify port/platform being tested (e.g. mac)"), | 85 help="Specify port/platform being tested (e.g.
mac)"), |
87 optparse.make_option("--chromium", | 86 optparse.make_option("--chromium", |
88 action="store_const", const='chromium', dest='platform', help='A
lias for --platform=chromium'), | 87 action="store_const", const='chromium', dest='p
latform', help='Alias for --platform=chromium'), |
89 optparse.make_option("--android", | 88 optparse.make_option("--android", |
90 action="store_const", const='android', dest='platform', help='Al
ias for --platform=android'), | 89 action="store_const", const='android', dest='pl
atform', help='Alias for --platform=android'), |
91 optparse.make_option("--builder-name", | 90 optparse.make_option("--builder-name", |
92 help=("The name of the builder shown on the waterfall running th
is script e.g. google-mac-2.")), | 91 help=("The name of the builder shown on the wat
erfall running this script e.g. google-mac-2.")), |
93 optparse.make_option("--build-number", | 92 optparse.make_option("--build-number", |
94 help=("The build number of the builder running this script.")), | 93 help=("The build number of the builder running
this script.")), |
95 optparse.make_option("--build", dest="build", action="store_true", d
efault=True, | 94 optparse.make_option("--build", dest="build", action="store_true", d
efault=True, |
96 help="Check to ensure the DumpRenderTree build is up-to-date (de
fault)."), | 95 help="Check to ensure the DumpRenderTree build
is up-to-date (default)."), |
97 optparse.make_option("--no-build", dest="build", action="store_false
", | 96 optparse.make_option("--no-build", dest="build", action="store_false
", |
98 help="Don't check to see if the DumpRenderTree build is up-to-da
te."), | 97 help="Don't check to see if the DumpRenderTree
build is up-to-date."), |
99 optparse.make_option("--build-directory", | 98 optparse.make_option("--build-directory", |
100 help="Path to the directory under which build files are kept (sh
ould not include configuration)"), | 99 help="Path to the directory under which build f
iles are kept (should not include configuration)"), |
101 optparse.make_option("--time-out-ms", default=600 * 1000, | 100 optparse.make_option("--time-out-ms", default=600 * 1000, |
102 help="Set the timeout for each test"), | 101 help="Set the timeout for each test"), |
103 optparse.make_option("--no-results", action="store_false", dest="gen
erate_results", default=True, | 102 optparse.make_option("--no-results", action="store_false", dest="gen
erate_results", default=True, |
104 help="Do no generate results JSON and results page."), | 103 help="Do no generate results JSON and results p
age."), |
105 optparse.make_option("--output-json-path", action='callback', callba
ck=_expand_path, type="str", | 104 optparse.make_option("--output-json-path", action='callback', callba
ck=_expand_path, type="str", |
106 help="Path to generate a JSON file at; may contain previous resu
lts if it already exists."), | 105 help="Path to generate a JSON file at; may cont
ain previous results if it already exists."), |
107 optparse.make_option("--reset-results", action="store_true", | 106 optparse.make_option("--reset-results", action="store_true", |
108 help="Clears the content in the generated JSON file before addin
g the results."), | 107 help="Clears the content in the generated JSON
file before adding the results."), |
109 optparse.make_option("--slave-config-json-path", action='callback',
callback=_expand_path, type="str", | 108 optparse.make_option("--slave-config-json-path", action='callback',
callback=_expand_path, type="str", |
110 help="Only used on bots. Path to a slave configuration file."), | 109 help="Only used on bots. Path to a slave config
uration file."), |
111 optparse.make_option("--description", | 110 optparse.make_option("--description", |
112 help="Add a description to the output JSON file if one is genera
ted"), | 111 help="Add a description to the output JSON file
if one is generated"), |
113 optparse.make_option("--no-show-results", action="store_false", defa
ult=True, dest="show_results", | 112 optparse.make_option("--no-show-results", action="store_false", defa
ult=True, dest="show_results", |
114 help="Don't launch a browser with results after the tests are do
ne"), | 113 help="Don't launch a browser with results after
the tests are done"), |
115 optparse.make_option("--test-results-server", | 114 optparse.make_option("--test-results-server", |
116 help="Upload the generated JSON file to the specified server whe
n --output-json-path is present."), | 115 help="Upload the generated JSON file to the spe
cified server when --output-json-path is present."), |
117 optparse.make_option("--force", dest="use_skipped_list", action="sto
re_false", default=True, | 116 optparse.make_option("--force", dest="use_skipped_list", action="sto
re_false", default=True, |
118 help="Run all tests, including the ones in the Skipped list."), | 117 help="Run all tests, including the ones in the
Skipped list."), |
119 optparse.make_option("--profile", action="store_true", | 118 optparse.make_option("--profile", action="store_true", |
120 help="Output per-test profile information."), | 119 help="Output per-test profile information."), |
121 optparse.make_option("--profiler", action="store", | 120 optparse.make_option("--profiler", action="store", |
122 help="Output per-test profile information, using the specified p
rofiler."), | 121 help="Output per-test profile information, usin
g the specified profiler."), |
123 optparse.make_option("--additional-driver-flag", action="append", | 122 optparse.make_option("--additional-driver-flag", action="append", |
124 default=[], help="Additional command line flag to pass to DumpRe
nderTree " | 123 default=[], help="Additional command line flag
to pass to DumpRenderTree " |
125 "Specify multiple times to add multiple flags."), | 124 "Specify multiple times to add multiple flags."
), |
126 optparse.make_option("--driver-name", type="string", | 125 optparse.make_option("--driver-name", type="string", |
127 help="Alternative DumpRenderTree binary to use"), | 126 help="Alternative DumpRenderTree binary to use"
), |
128 optparse.make_option("--content-shell", action="store_true", | 127 optparse.make_option("--content-shell", action="store_true", |
129 help="Use Content Shell instead of DumpRenderTree"), | 128 help="Use Content Shell instead of DumpRenderTr
ee"), |
130 optparse.make_option("--repeat", default=1, type="int", | 129 optparse.make_option("--repeat", default=1, type="int", |
131 help="Specify number of times to run test set (default: 1)."), | 130 help="Specify number of times to run test set (
default: 1)."), |
132 optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUN
NER_COUNT, type="int", | 131 optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUN
NER_COUNT, type="int", |
133 help="Specify number of times to invoke test runner for each per
formance test."), | 132 help="Specify number of times to invoke test ru
nner for each performance test."), |
134 ] | 133 ] |
135 return optparse.OptionParser(option_list=(perf_option_list)).parse_args(
args) | 134 return optparse.OptionParser(option_list=(perf_option_list)).parse_args(
args) |
136 | 135 |
137 def _collect_tests(self): | 136 def _collect_tests(self): |
138 test_extensions = ['.html', '.svg'] | 137 test_extensions = ['.html', '.svg'] |
139 | 138 |
140 def _is_test_file(filesystem, dirname, filename): | 139 def _is_test_file(filesystem, dirname, filename): |
141 return filesystem.splitext(filename)[1] in test_extensions | 140 return filesystem.splitext(filename)[1] in test_extensions |
142 | 141 |
143 filesystem = self._host.filesystem | 142 filesystem = self._host.filesystem |
144 | 143 |
145 paths = [] | 144 paths = [] |
146 for arg in self._args: | 145 for arg in self._args: |
147 if filesystem.exists(filesystem.join(self._base_path, arg)): | 146 if filesystem.exists(filesystem.join(self._base_path, arg)): |
148 paths.append(arg) | 147 paths.append(arg) |
149 else: | 148 else: |
150 relpath = filesystem.relpath(arg, self._base_path) | 149 relpath = filesystem.relpath(arg, self._base_path) |
151 if filesystem.exists(filesystem.join(self._base_path, relpath)): | 150 if filesystem.exists(filesystem.join(self._base_path, relpath)): |
152 paths.append(filesystem.normpath(relpath)) | 151 paths.append(filesystem.normpath(relpath)) |
153 else: | 152 else: |
154 _log.warn('Path was not found:' + arg) | 153 _log.warn('Path was not found:' + arg) |
155 | 154 |
156 skipped_directories = set(['.svn', 'resources']) | 155 skipped_directories = set(['.svn', 'resources']) |
157 test_files = find_files.find(filesystem, self._base_path, paths, skipped
_directories, _is_test_file) | 156 test_files = find_files.find(filesystem, self._base_path, paths, skipped
_directories, _is_test_file) |
158 tests = [] | 157 tests = [] |
159 for path in test_files: | 158 for path in test_files: |
160 relative_path = filesystem.relpath(path, self._base_path).replace('\
\', '/') | 159 relative_path = filesystem.relpath(path, self._base_path).replace('\
\', '/') |
161 if self._options.use_skipped_list and self._port.skips_perf_test(rel
ative_path) and filesystem.normpath(relative_path) not in paths: | 160 if self._options.use_skipped_list and self._port.skips_perf_test(rel
ative_path) and filesystem.normpath(relative_path) not in paths: |
162 continue | 161 continue |
163 test = PerfTestFactory.create_perf_test(self._port, relative_path, p
ath, test_runner_count=self._options.test_runner_count) | 162 test = PerfTestFactory.create_perf_test(self._port, relative_path, p
ath, |
| 163 test_runner_count=self._opti
ons.test_runner_count) |
164 tests.append(test) | 164 tests.append(test) |
165 | 165 |
166 return tests | 166 return tests |
167 | 167 |
168 def _start_http_servers(self): | 168 def _start_http_servers(self): |
169 self._port.acquire_http_lock() | 169 self._port.acquire_http_lock() |
170 self._port.start_http_server(number_of_servers=2) | 170 self._port.start_http_server(number_of_servers=2) |
171 | 171 |
172 def _stop_http_servers(self): | 172 def _stop_http_servers(self): |
173 self._port.stop_http_server() | 173 self._port.stop_http_server() |
174 self._port.release_http_lock() | 174 self._port.release_http_lock() |
175 | 175 |
176 def run(self): | 176 def run(self): |
177 needs_http = self._port.requires_http_server() | 177 needs_http = self._port.requires_http_server() |
178 | 178 |
179 class FakePrinter(object): | 179 class FakePrinter(object): |
| 180 |
180 def write_update(self, msg): | 181 def write_update(self, msg): |
181 print msg | 182 print msg |
182 | 183 |
183 def write_throttled_update(self, msg): | 184 def write_throttled_update(self, msg): |
184 pass | 185 pass |
185 | 186 |
186 if self._port.check_build(needs_http=needs_http, printer=FakePrinter()): | 187 if self._port.check_build(needs_http=needs_http, printer=FakePrinter()): |
187 _log.error("Build not up to date for %s" % self._port._path_to_drive
r()) | 188 _log.error("Build not up to date for %s" % self._port._path_to_drive
r()) |
188 return self.EXIT_CODE_BAD_BUILD | 189 return self.EXIT_CODE_BAD_BUILD |
189 | 190 |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
229 if output_json_path: | 230 if output_json_path: |
230 return output_json_path | 231 return output_json_path |
231 return self._host.filesystem.join(self._port.perf_results_directory(), s
elf._DEFAULT_JSON_FILENAME) | 232 return self._host.filesystem.join(self._port.perf_results_directory(), s
elf._DEFAULT_JSON_FILENAME) |
232 | 233 |
233 def _results_page_path(self): | 234 def _results_page_path(self): |
234 return self._host.filesystem.splitext(self._output_json_path())[0] + '.h
tml' | 235 return self._host.filesystem.splitext(self._output_json_path())[0] + '.h
tml' |
235 | 236 |
236 def _generate_results(self): | 237 def _generate_results(self): |
237 options = self._options | 238 options = self._options |
238 output_json_path = self._output_json_path() | 239 output_json_path = self._output_json_path() |
239 output = self._generate_results_dict(self._timestamp, options.descriptio
n, options.platform, options.builder_name, options.build_number) | 240 output = self._generate_results_dict(self._timestamp, options.descriptio
n, |
| 241 options.platform, options.builder_n
ame, options.build_number) |
240 | 242 |
241 if options.slave_config_json_path: | 243 if options.slave_config_json_path: |
242 output = self._merge_slave_config_json(options.slave_config_json_pat
h, output) | 244 output = self._merge_slave_config_json(options.slave_config_json_pat
h, output) |
243 if not output: | 245 if not output: |
244 return self.EXIT_CODE_BAD_SOURCE_JSON | 246 return self.EXIT_CODE_BAD_SOURCE_JSON |
245 | 247 |
246 output = self._merge_outputs_if_needed(output_json_path, output) | 248 output = self._merge_outputs_if_needed(output_json_path, output) |
247 if not output: | 249 if not output: |
248 return self.EXIT_CODE_BAD_MERGE | 250 return self.EXIT_CODE_BAD_MERGE |
249 | 251 |
(...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
369 if metrics: | 371 if metrics: |
370 self._results.append((test, metrics)) | 372 self._results.append((test, metrics)) |
371 else: | 373 else: |
372 failures += 1 | 374 failures += 1 |
373 _log.error('FAILED') | 375 _log.error('FAILED') |
374 | 376 |
375 _log.info('Finished: %f s' % (time.time() - start_time)) | 377 _log.info('Finished: %f s' % (time.time() - start_time)) |
376 _log.info('') | 378 _log.info('') |
377 | 379 |
378 return failures | 380 return failures |
OLD | NEW |