| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged | 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged |
| 3 # | 3 # |
| 4 # Redistribution and use in source and binary forms, with or without | 4 # Redistribution and use in source and binary forms, with or without |
| 5 # modification, are permitted provided that the following conditions are | 5 # modification, are permitted provided that the following conditions are |
| 6 # met: | 6 # met: |
| 7 # | 7 # |
| 8 # * Redistributions of source code must retain the above copyright | 8 # * Redistributions of source code must retain the above copyright |
| 9 # notice, this list of conditions and the following disclaimer. | 9 # notice, this list of conditions and the following disclaimer. |
| 10 # * Redistributions in binary form must reproduce the above | 10 # * Redistributions in binary form must reproduce the above |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 142 tests_to_retry = self._tests_to_retry(initial_results) | 142 tests_to_retry = self._tests_to_retry(initial_results) |
| 143 all_retry_results = [] | 143 all_retry_results = [] |
| 144 if should_retry_failures and tests_to_retry: | 144 if should_retry_failures and tests_to_retry: |
| 145 enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed
() | 145 enabled_pixel_tests_in_retry = self._force_pixel_tests_if_needed
() |
| 146 | 146 |
| 147 for retry_attempt in xrange(1, self._options.num_retries + 1): | 147 for retry_attempt in xrange(1, self._options.num_retries + 1): |
| 148 if not tests_to_retry: | 148 if not tests_to_retry: |
| 149 break | 149 break |
| 150 | 150 |
| 151 _log.info('') | 151 _log.info('') |
| 152 _log.info('Retrying %s, attempt %d of %d...' % | 152 _log.info('Retrying %s, attempt %d of %d...', |
| 153 (grammar.pluralize('unexpected failure', len(tests
_to_retry)), | 153 grammar.pluralize('unexpected failure', len(tests_
to_retry)), |
| 154 retry_attempt, self._options.num_retries)) | 154 retry_attempt, self._options.num_retries) |
| 155 | 155 |
| 156 retry_results = self._run_tests(tests_to_retry, | 156 retry_results = self._run_tests(tests_to_retry, |
| 157 tests_to_skip=set(), | 157 tests_to_skip=set(), |
| 158 repeat_each=1, | 158 repeat_each=1, |
| 159 iterations=1, | 159 iterations=1, |
| 160 num_workers=num_workers, | 160 num_workers=num_workers, |
| 161 retry_attempt=retry_attempt) | 161 retry_attempt=retry_attempt) |
| 162 all_retry_results.append(retry_results) | 162 all_retry_results.append(retry_results) |
| 163 | 163 |
| 164 tests_to_retry = self._tests_to_retry(retry_results) | 164 tests_to_retry = self._tests_to_retry(retry_results) |
| (...skipping 14 matching lines...) Expand all Loading... |
| 179 _log.debug("summarizing results") | 179 _log.debug("summarizing results") |
| 180 summarized_full_results = test_run_results.summarize_results( | 180 summarized_full_results = test_run_results.summarize_results( |
| 181 self._port, self._expectations, initial_results, all_retry_results, | 181 self._port, self._expectations, initial_results, all_retry_results, |
| 182 enabled_pixel_tests_in_retry) | 182 enabled_pixel_tests_in_retry) |
| 183 summarized_failing_results = test_run_results.summarize_results( | 183 summarized_failing_results = test_run_results.summarize_results( |
| 184 self._port, self._expectations, initial_results, all_retry_results, | 184 self._port, self._expectations, initial_results, all_retry_results, |
| 185 enabled_pixel_tests_in_retry, only_include_failing=True) | 185 enabled_pixel_tests_in_retry, only_include_failing=True) |
| 186 | 186 |
| 187 exit_code = summarized_failing_results['num_regressions'] | 187 exit_code = summarized_failing_results['num_regressions'] |
| 188 if exit_code > test_run_results.MAX_FAILURES_EXIT_STATUS: | 188 if exit_code > test_run_results.MAX_FAILURES_EXIT_STATUS: |
| 189 _log.warning('num regressions (%d) exceeds max exit status (%d)' % | 189 _log.warning('num regressions (%d) exceeds max exit status (%d)', |
| 190 (exit_code, test_run_results.MAX_FAILURES_EXIT_STATUS)) | 190 exit_code, test_run_results.MAX_FAILURES_EXIT_STATUS) |
| 191 exit_code = test_run_results.MAX_FAILURES_EXIT_STATUS | 191 exit_code = test_run_results.MAX_FAILURES_EXIT_STATUS |
| 192 | 192 |
| 193 if not self._options.dry_run: | 193 if not self._options.dry_run: |
| 194 self._write_json_files(summarized_full_results, summarized_failing_r
esults, initial_results, running_all_tests) | 194 self._write_json_files(summarized_full_results, summarized_failing_r
esults, initial_results, running_all_tests) |
| 195 | 195 |
| 196 if self._options.write_full_results_to: | 196 if self._options.write_full_results_to: |
| 197 self._filesystem.copyfile(self._filesystem.join(self._results_di
rectory, "full_results.json"), | 197 self._filesystem.copyfile(self._filesystem.join(self._results_di
rectory, "full_results.json"), |
| 198 self._options.write_full_results_to) | 198 self._options.write_full_results_to) |
| 199 | 199 |
| 200 self._upload_json_files() | 200 self._upload_json_files() |
| (...skipping 269 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 470 # Port specific clean-up. | 470 # Port specific clean-up. |
| 471 self._port.clobber_old_port_specific_results() | 471 self._port.clobber_old_port_specific_results() |
| 472 | 472 |
| 473 def _tests_to_retry(self, run_results): | 473 def _tests_to_retry(self, run_results): |
| 474 # TODO(ojan): This should also check that result.type != test_expectatio
ns.MISSING since retrying missing expectations is silly. | 474 # TODO(ojan): This should also check that result.type != test_expectatio
ns.MISSING since retrying missing expectations is silly. |
| 475 # But that's a bit tricky since we only consider the last retry attempt
for the count of unexpected regressions. | 475 # But that's a bit tricky since we only consider the last retry attempt
for the count of unexpected regressions. |
| 476 return [result.test_name for result in run_results.unexpected_results_by
_name.values( | 476 return [result.test_name for result in run_results.unexpected_results_by
_name.values( |
| 477 ) if result.type != test_expectations.PASS] | 477 ) if result.type != test_expectations.PASS] |
| 478 | 478 |
| 479 def _write_json_files(self, summarized_full_results, summarized_failing_resu
lts, initial_results, running_all_tests): | 479 def _write_json_files(self, summarized_full_results, summarized_failing_resu
lts, initial_results, running_all_tests): |
| 480 _log.debug("Writing JSON files in %s." % self._results_directory) | 480 _log.debug("Writing JSON files in %s.", self._results_directory) |
| 481 | 481 |
| 482 # FIXME: Upload stats.json to the server and delete times_ms. | 482 # FIXME: Upload stats.json to the server and delete times_ms. |
| 483 times_trie = json_results_generator.test_timings_trie(initial_results.re
sults_by_name.values()) | 483 times_trie = json_results_generator.test_timings_trie(initial_results.re
sults_by_name.values()) |
| 484 times_json_path = self._filesystem.join(self._results_directory, "times_
ms.json") | 484 times_json_path = self._filesystem.join(self._results_directory, "times_
ms.json") |
| 485 json_results_generator.write_json(self._filesystem, times_trie, times_js
on_path) | 485 json_results_generator.write_json(self._filesystem, times_trie, times_js
on_path) |
| 486 | 486 |
| 487 # Save out the times data so we can use it for --fastest in the future. | 487 # Save out the times data so we can use it for --fastest in the future. |
| 488 if running_all_tests: | 488 if running_all_tests: |
| 489 bot_test_times_path = self._port.bot_test_times_path() | 489 bot_test_times_path = self._port.bot_test_times_path() |
| 490 self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_t
est_times_path)) | 490 self._filesystem.maybe_make_directory(self._filesystem.dirname(bot_t
est_times_path)) |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 526 url = "http://%s/testfile/upload" % self._options.test_results_server | 526 url = "http://%s/testfile/upload" % self._options.test_results_server |
| 527 # Set uploading timeout in case appengine server is having problems. | 527 # Set uploading timeout in case appengine server is having problems. |
| 528 # 120 seconds are more than enough to upload test results. | 528 # 120 seconds are more than enough to upload test results. |
| 529 uploader = FileUploader(url, 120) | 529 uploader = FileUploader(url, 120) |
| 530 try: | 530 try: |
| 531 response = uploader.upload_as_multipart_form_data(self._filesystem,
files, attrs) | 531 response = uploader.upload_as_multipart_form_data(self._filesystem,
files, attrs) |
| 532 if response: | 532 if response: |
| 533 if response.code == 200: | 533 if response.code == 200: |
| 534 _log.debug("JSON uploaded.") | 534 _log.debug("JSON uploaded.") |
| 535 else: | 535 else: |
| 536 _log.debug("JSON upload failed, %d: '%s'" % (response.code,
response.read())) | 536 _log.debug("JSON upload failed, %d: '%s'", response.code, re
sponse.read()) |
| 537 else: | 537 else: |
| 538 _log.error("JSON upload failed; no response returned") | 538 _log.error("JSON upload failed; no response returned") |
| 539 except Exception as err: | 539 except Exception as err: |
| 540 _log.error("Upload failed: %s" % err) | 540 _log.error("Upload failed: %s", err) |
| 541 | 541 |
| 542 def _copy_results_html_file(self, destination_path): | 542 def _copy_results_html_file(self, destination_path): |
| 543 base_dir = self._port.path_from_webkit_base('LayoutTests', 'fast', 'harn
ess') | 543 base_dir = self._port.path_from_webkit_base('LayoutTests', 'fast', 'harn
ess') |
| 544 results_file = self._filesystem.join(base_dir, 'results.html') | 544 results_file = self._filesystem.join(base_dir, 'results.html') |
| 545 # Note that the results.html template file won't exist when we're using
a MockFileSystem during unit tests, | 545 # Note that the results.html template file won't exist when we're using
a MockFileSystem during unit tests, |
| 546 # so make sure it exists before we try to copy it. | 546 # so make sure it exists before we try to copy it. |
| 547 if self._filesystem.exists(results_file): | 547 if self._filesystem.exists(results_file): |
| 548 self._filesystem.copyfile(results_file, destination_path) | 548 self._filesystem.copyfile(results_file, destination_path) |
| 549 | 549 |
| 550 def _stats_trie(self, initial_results): | 550 def _stats_trie(self, initial_results): |
| 551 def _worker_number(worker_name): | 551 def _worker_number(worker_name): |
| 552 return int(worker_name.split('/')[1]) if worker_name else -1 | 552 return int(worker_name.split('/')[1]) if worker_name else -1 |
| 553 | 553 |
| 554 stats = {} | 554 stats = {} |
| 555 for result in initial_results.results_by_name.values(): | 555 for result in initial_results.results_by_name.values(): |
| 556 if result.type != test_expectations.SKIP: | 556 if result.type != test_expectations.SKIP: |
| 557 stats[result.test_name] = {'results': (_worker_number(result.wor
ker_name), result.test_number, result.pid, int( | 557 stats[result.test_name] = {'results': (_worker_number(result.wor
ker_name), result.test_number, result.pid, int( |
| 558 result.test_run_time * 1000), int(result.total_run_time * 10
00))} | 558 result.test_run_time * 1000), int(result.total_run_time * 10
00))} |
| 559 stats_trie = {} | 559 stats_trie = {} |
| 560 for name, value in stats.iteritems(): | 560 for name, value in stats.iteritems(): |
| 561 json_results_generator.add_path_to_trie(name, value, stats_trie) | 561 json_results_generator.add_path_to_trie(name, value, stats_trie) |
| 562 return stats_trie | 562 return stats_trie |
| OLD | NEW |