| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Updates layout test expectations and baselines when updating w3c tests. | 5 """Updates layout test expectations and baselines when updating w3c tests. |
| 6 | 6 |
| 7 Specifically, this class fetches results from try bots for the current CL, then | 7 Specifically, this class fetches results from try bots for the current CL, then |
| 8 (1) downloads new baseline files for any tests that can be rebaselined, and | 8 (1) downloads new baseline files for any tests that can be rebaselined, and |
| 9 (2) updates the generic TestExpectations file for any other failing tests. | 9 (2) updates the generic TestExpectations file for any other failing tests. |
| 10 """ | 10 """ |
| 11 | 11 |
| 12 import argparse | 12 import argparse |
| 13 import copy | 13 import copy |
| 14 import logging | 14 import logging |
| 15 | 15 |
| 16 from webkitpy.common.memoized import memoized | 16 from webkitpy.common.memoized import memoized |
| 17 from webkitpy.common.net.git_cl import GitCL | 17 from webkitpy.common.net.git_cl import GitCL |
| 18 from webkitpy.common.net.rietveld import Rietveld | |
| 19 from webkitpy.common.webkit_finder import WebKitFinder | 18 from webkitpy.common.webkit_finder import WebKitFinder |
| 20 from webkitpy.layout_tests.models.test_expectations import TestExpectationLine,
TestExpectations | 19 from webkitpy.layout_tests.models.test_expectations import TestExpectationLine,
TestExpectations |
| 21 from webkitpy.w3c.test_parser import TestParser | 20 from webkitpy.w3c.test_parser import TestParser |
| 22 | 21 |
| 23 _log = logging.getLogger(__name__) | 22 _log = logging.getLogger(__name__) |
| 24 | 23 |
| 25 MARKER_COMMENT = '# ====== New tests from w3c-test-autoroller added here ======' | 24 MARKER_COMMENT = '# ====== New tests from w3c-test-autoroller added here ======' |
| 26 | 25 |
| 27 | 26 |
| 28 class WPTExpectationsUpdater(object): | 27 class WPTExpectationsUpdater(object): |
| 29 | 28 |
| 30 def __init__(self, host): | 29 def __init__(self, host): |
| 31 self.host = host | 30 self.host = host |
| 32 self.finder = WebKitFinder(self.host.filesystem) | 31 self.finder = WebKitFinder(self.host.filesystem) |
| 33 | 32 |
| 34 def run(self, args=None): | 33 def run(self, args=None): |
| 35 """Downloads text new baselines and adds test expectations lines.""" | 34 """Downloads text new baselines and adds test expectations lines.""" |
| 36 parser = argparse.ArgumentParser(description=__doc__) | 35 parser = argparse.ArgumentParser(description=__doc__) |
| 37 parser.add_argument('-v', '--verbose', action='store_true', help='More v
erbose logging.') | 36 parser.add_argument('-v', '--verbose', action='store_true', help='More v
erbose logging.') |
| 38 args = parser.parse_args(args) | 37 args = parser.parse_args(args) |
| 39 | 38 |
| 40 log_level = logging.DEBUG if args.verbose else logging.INFO | 39 log_level = logging.DEBUG if args.verbose else logging.INFO |
| 41 logging.basicConfig(level=log_level, format='%(message)s') | 40 logging.basicConfig(level=log_level, format='%(message)s') |
| 42 | 41 |
| 43 issue_number = self.get_issue_number() | 42 issue_number = self.get_issue_number() |
| 44 if issue_number == 'None': | 43 if issue_number == 'None': |
| 45 _log.error('No issue on current branch.') | 44 _log.error('No issue on current branch.') |
| 46 return 1 | 45 return 1 |
| 47 | 46 |
| 48 rietveld = Rietveld(self.host.web) | 47 builds = self.get_latest_try_jobs() |
| 49 builds = rietveld.latest_try_jobs(issue_number, self._get_try_bots()) | |
| 50 _log.debug('Latest try jobs: %r', builds) | 48 _log.debug('Latest try jobs: %r', builds) |
| 51 if not builds: | 49 if not builds: |
| 52 _log.error('No try job information was collected.') | 50 _log.error('No try job information was collected.') |
| 53 return 1 | 51 return 1 |
| 54 | 52 |
| 55 # Here we build up a dict of failing test results for all platforms. | 53 # Here we build up a dict of failing test results for all platforms. |
| 56 test_expectations = {} | 54 test_expectations = {} |
| 57 for build in builds: | 55 for build in builds: |
| 58 port_results = self.get_failing_results_dict(build) | 56 port_results = self.get_failing_results_dict(build) |
| 59 test_expectations = self.merge_dicts(test_expectations, port_results
) | 57 test_expectations = self.merge_dicts(test_expectations, port_results
) |
| 60 | 58 |
| 61 # And then we merge results for different platforms that had the same re
sults. | 59 # And then we merge results for different platforms that had the same re
sults. |
| 62 for test_name, platform_result in test_expectations.iteritems(): | 60 for test_name, platform_result in test_expectations.iteritems(): |
| 63 # platform_result is a dict mapping platforms to results. | 61 # platform_result is a dict mapping platforms to results. |
| 64 test_expectations[test_name] = self.merge_same_valued_keys(platform_
result) | 62 test_expectations[test_name] = self.merge_same_valued_keys(platform_
result) |
| 65 | 63 |
| 66 test_expectations = self.download_text_baselines(test_expectations) | 64 test_expectations = self.download_text_baselines(test_expectations) |
| 67 test_expectation_lines = self.create_line_list(test_expectations) | 65 test_expectation_lines = self.create_line_list(test_expectations) |
| 68 self.write_to_test_expectations(test_expectation_lines) | 66 self.write_to_test_expectations(test_expectation_lines) |
| 69 return 0 | 67 return 0 |
| 70 | 68 |
| 71 def get_issue_number(self): | 69 def get_issue_number(self): |
| 72 """Returns current CL number. Can be replaced in unit tests.""" | 70 """Returns current CL number. Can be replaced in unit tests.""" |
| 73 return GitCL(self.host).get_issue_number() | 71 return GitCL(self.host).get_issue_number() |
| 74 | 72 |
| 73 def get_latest_try_jobs(self): |
| 74 """Returns the latest finished try jobs as Build objects.""" |
| 75 return GitCL(self.host).latest_try_jobs(self._get_try_bots()) |
| 76 |
| 75 def get_failing_results_dict(self, build): | 77 def get_failing_results_dict(self, build): |
| 76 """Returns a nested dict of failing test results. | 78 """Returns a nested dict of failing test results. |
| 77 | 79 |
| 78 Retrieves a full list of layout test results from a builder result URL. | 80 Retrieves a full list of layout test results from a builder result URL. |
| 79 Collects the builder name, platform and a list of tests that did not | 81 Collects the builder name, platform and a list of tests that did not |
| 80 run as expected. | 82 run as expected. |
| 81 | 83 |
| 82 Args: | 84 Args: |
| 83 build: A Build object. | 85 build: A Build object. |
| 84 | 86 |
| (...skipping 351 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 436 This might correspond to a deleted file or a non-test. | 438 This might correspond to a deleted file or a non-test. |
| 437 """ | 439 """ |
| 438 absolute_path = self.host.filesystem.join(self.finder.layout_tests_dir()
, test_path) | 440 absolute_path = self.host.filesystem.join(self.finder.layout_tests_dir()
, test_path) |
| 439 test_parser = TestParser(absolute_path, self.host) | 441 test_parser = TestParser(absolute_path, self.host) |
| 440 if not test_parser.test_doc: | 442 if not test_parser.test_doc: |
| 441 return False | 443 return False |
| 442 return test_parser.is_jstest() | 444 return test_parser.is_jstest() |
| 443 | 445 |
| 444 def _get_try_bots(self): | 446 def _get_try_bots(self): |
| 445 return self.host.builders.all_try_builder_names() | 447 return self.host.builders.all_try_builder_names() |
| OLD | NEW |